Skip to content

Commit

Permalink
fix mirroring for dual cephblockpools
Browse files Browse the repository at this point in the history 
fix rbd mirroring and peer secrets addition, when more than
one cephblockpool is configured in ramen tests

Signed-off-by: Nikhil-Ladha <[email protected]>
  • Loading branch information
@Nikhil-Ladha
Nikhil-Ladha committed Feb 13, 2025
1 parent ca44e13 commit 28732b9
Show file tree
Hide file tree
Showing 2 changed files with 113 additions and 103 deletions.
150 changes: 79 additions & 71 deletions test/addons/rbd-mirror/start
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ from drenv import ceph
from drenv import commands
from drenv import kubectl

POOL_NAME = "replicapool"
POOL_NAMES = ["replicapool", "replicapool-2"]


def log_blocklist(cluster):
Expand All @@ -32,33 +32,35 @@ def fetch_secret_info(cluster):
info = {}

print(f"Getting mirroring info site name for cluster '{cluster}'")
info["name"] = drenv.wait_for(
f"cephblockpools.ceph.rook.io/{POOL_NAME}",
output="jsonpath={.status.mirroringInfo.site_name}",
namespace="rook-ceph",
profile=cluster,
)
for pool in POOL_NAMES:
info[pool] = {}
info[pool]["name"] = drenv.wait_for(
f"cephblockpools.ceph.rook.io/{pool}",
output="jsonpath={.status.mirroringInfo.site_name}",
namespace="rook-ceph",
profile=cluster,
)

print(f"Getting rbd mirror boostrap peer secret name for cluster '{cluster}'")
secret_name = kubectl.get(
"cephblockpools.ceph.rook.io",
POOL_NAME,
"--output=jsonpath={.status.info.rbdMirrorBootstrapPeerSecretName}",
"--namespace=rook-ceph",
context=cluster,
)
print(f"Getting rbd mirror boostrap peer secret name for cluster '{cluster}'")
secret_name = kubectl.get(
"cephblockpools.ceph.rook.io",
pool,
"--output=jsonpath={.status.info.rbdMirrorBootstrapPeerSecretName}",
"--namespace=rook-ceph",
context=cluster,
)

print(f"Getting secret {secret_name} token for cluster '{cluster}'")
info["token"] = kubectl.get(
"secret",
secret_name,
"--output=jsonpath={.data.token}",
"--namespace=rook-ceph",
context=cluster,
)
print(f"Getting secret {secret_name} token for cluster '{cluster}'")
info[pool]["token"] = kubectl.get(
"secret",
secret_name,
"--output=jsonpath={.data.token}",
"--namespace=rook-ceph",
context=cluster,
)

# Must be encoded as base64 in secret .data section.
info["pool"] = base64.b64encode(POOL_NAME.encode()).decode()
# Must be encoded as base64 in secret .data section.
info[pool]["pool"] = base64.b64encode(pool.encode()).decode()

return info

Expand Down Expand Up @@ -86,33 +88,37 @@ def disable_rbd_mirror_debug_logs(cluster):
def configure_rbd_mirroring(cluster, peer_info):
print(f"Applying rbd mirror secret in cluster '{cluster}'")

template = drenv.template("start-data/rbd-mirror-secret.yaml")
yaml = template.substitute(peer_info)
kubectl.apply(
"--filename=-",
"--namespace=rook-ceph",
input=yaml,
context=cluster,
)
for pool in POOL_NAMES:
template = drenv.template("start-data/rbd-mirror-secret.yaml")
yaml = template.substitute(peer_info[pool])
kubectl.apply(
"--filename=-",
"--namespace=rook-ceph",
input=yaml,
context=cluster,
)

print(f"Configure peers for cluster '{cluster}'")
patch = {"spec": {"mirroring": {"peers": {"secretNames": [peer_info["name"]]}}}}
kubectl.patch(
"cephblockpool",
POOL_NAME,
"--type=merge",
f"--patch={json.dumps(patch)}",
"--namespace=rook-ceph",
context=cluster,
)
print(f"Configure peers for cluster '{cluster}'")
# pool = peer
patch = {
"spec": {"mirroring": {"peers": {"secretNames": [peer_info[pool]["name"]]}}}
}
kubectl.patch(
"cephblockpool",
pool,
"--type=merge",
f"--patch={json.dumps(patch)}",
"--namespace=rook-ceph",
context=cluster,
)

print("Creating VolumeReplicationClass")
template = drenv.template("start-data/vrc-sample.yaml")
yaml = template.substitute(cluster=cluster)
kubectl.apply("--filename=-", input=yaml, context=cluster)

template = drenv.template("start-data/vgrc-sample.yaml")
yaml = template.substitute(cluster=cluster, pool=POOL_NAME)
yaml = template.substitute(cluster=cluster, pool="replicapool")
kubectl.apply("--filename=-", input=yaml, context=cluster)

print(f"Apply rbd mirror to cluster '{cluster}'")
Expand Down Expand Up @@ -161,15 +167,16 @@ def wait_until_pool_mirroring_is_healthy(cluster, attempts=3):
else:
break

status = kubectl.get(
"cephblockpools.ceph.rook.io",
POOL_NAME,
"--output=jsonpath={.status}",
"--namespace=rook-ceph",
context=cluster,
)
info = {f"Cluster '{cluster}' ceph block pool status": json.loads(status)}
print(yaml.dump(info, sort_keys=False))
for pool in POOL_NAMES:
status = kubectl.get(
"cephblockpools.ceph.rook.io",
pool,
"--output=jsonpath={.status}",
"--namespace=rook-ceph",
context=cluster,
)
info = {f"Cluster '{cluster}' ceph block pool status": json.loads(status)}
print(yaml.dump(info, sort_keys=False))


def watch_pool_mirroring_status(cluster, timeout=180):
Expand All @@ -182,24 +189,25 @@ def watch_pool_mirroring_status(cluster, timeout=180):

while True:
remaining = deadline - time.monotonic()
watcher = kubectl.watch(
f"cephblockpool/{POOL_NAME}",
jsonpath="{.status.mirroringStatus.summary}",
namespace="rook-ceph",
timeout=remaining,
context=cluster,
)
with closing(watcher):
for line in watcher:
summary = json.loads(line)
print(f"Cluster '{cluster}' mirroring status': {summary}")

if mirroring_is_healthy(summary):
elapsed = time.monotonic() - start
print(
f"Cluster '{cluster}' mirroring healthy in {elapsed:.2f} seconds"
)
return
for pool in POOL_NAMES:
watcher = kubectl.watch(
f"cephblockpool/{pool}",
jsonpath="{.status.mirroringStatus.summary}",
namespace="rook-ceph",
timeout=remaining,
context=cluster,
)
with closing(watcher):
for line in watcher:
summary = json.loads(line)
print(f"Cluster '{cluster}' mirroring status': {summary}")

if mirroring_is_healthy(summary):
elapsed = time.monotonic() - start
print(
f"Cluster '{cluster}' mirroring healthy in {elapsed:.2f} seconds"
)
return


def mirroring_is_healthy(summary):
Expand Down
66 changes: 34 additions & 32 deletions test/addons/rook-pool/start
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,38 +43,40 @@ def deploy(cluster):

def wait(cluster):
print("Waiting until ceph block pool is ready")
drenv.wait_for(
"cephblockpool/replicapool",
output="jsonpath={.status.phase}",
namespace="rook-ceph",
timeout=120,
profile=cluster,
)
kubectl.wait(
"cephblockpool/replicapool",
"--for=jsonpath={.status.phase}=Ready",
"--namespace=rook-ceph",
"--timeout=300s",
context=cluster,
)

print("Waiting for replica pool peer token")
kubectl.wait(
"cephblockpool/replicapool",
"--for=jsonpath={.status.info.rbdMirrorBootstrapPeerSecretName}=pool-peer-token-replicapool",
"--namespace=rook-ceph",
"--timeout=300s",
context=cluster,
)

out = kubectl.get(
"cephblockpool/replicapool",
"--output=jsonpath={.status}",
"--namespace=rook-ceph",
context=cluster,
)
info = {"ceph pool status": json.loads(out)}
print(yaml.dump(info, sort_keys=False))
for pool in POOL_NAMES:
drenv.wait_for(
f"cephblockpool/{pool}",
output="jsonpath={.status.phase}",
namespace="rook-ceph",
timeout=120,
profile=cluster,
)
kubectl.wait(
f"cephblockpool/{pool}",
"--for=jsonpath={.status.phase}=Ready",
"--namespace=rook-ceph",
"--timeout=300s",
context=cluster,
)

print("Waiting for replica pool peer token")
kubectl.wait(
f"cephblockpool/{pool}",
"--for=jsonpath={.status.info.rbdMirrorBootstrapPeerSecretName}"
f"=pool-peer-token-{pool}",
"--namespace=rook-ceph",
"--timeout=300s",
context=cluster,
)

out = kubectl.get(
f"cephblockpool/{pool}",
"--output=jsonpath={.status}",
"--namespace=rook-ceph",
context=cluster,
)
info = {"ceph pool status": json.loads(out)}
print(yaml.dump(info, sort_keys=False))


if len(sys.argv) != 2:
Expand Down

0 comments on commit 28732b9

Please sign in to comment.