From 0a2fe22741beae4e955b461343ec981e65ea19dd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 09:21:08 -0500 Subject: [PATCH 01/10] Build(deps): Bump github/codeql-action from 3.27.7 to 3.27.9 (#15618) Bumps [github/codeql-action](https://github.com/github/codeql-action) from 3.27.7 to 3.27.9. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/babb554ede22fd5605947329c4d04d8e7a0b8155...df409f7d9260372bd5f19e5b04e83cb3c43714ae) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ossf-scorecard.yml | 2 +- .github/workflows/trivy.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ossf-scorecard.yml b/.github/workflows/ossf-scorecard.yml index 5117f992ebd..5c1f3b4054c 100644 --- a/.github/workflows/ossf-scorecard.yml +++ b/.github/workflows/ossf-scorecard.yml @@ -71,6 +71,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@babb554ede22fd5605947329c4d04d8e7a0b8155 # v3.27.7 + uses: github/codeql-action/upload-sarif@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9 with: sarif_file: results.sarif diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 45a2a0edd1e..cb8299d765e 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -58,7 +58,7 @@ jobs: trivy-config: 'utils/trivy/trivy.yaml' - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@babb554ede22fd5605947329c4d04d8e7a0b8155 # v3.27.7 + uses: github/codeql-action/upload-sarif@df409f7d9260372bd5f19e5b04e83cb3c43714ae # v3.27.9 with: sarif_file: 'trivy-results.sarif' From ad736a29d1cee446322976db89ab528d5ba50d3a Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Mon, 16 Dec 2024 07:07:09 -0800 Subject: [PATCH 02/10] DAOS-16873 cq: remove workflows/version-checks.yml (#15601) Remove workflows/version-checks.yml now that dependabot checks this. Signed-off-by: Dalton Bohning --- .github/workflows/version-checks.yml | 35 ---------------------------- 1 file changed, 35 deletions(-) delete mode 100644 .github/workflows/version-checks.yml diff --git a/.github/workflows/version-checks.yml b/.github/workflows/version-checks.yml deleted file mode 100644 index 6f9d4cc7327..00000000000 --- a/.github/workflows/version-checks.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Version checking - -on: - push: - branches: - - master - pull_request: - paths: - - 'utils/cq/requirements.txt' - - '.github/workflows/version-checks.yml' - -permissions: {} - -jobs: - upgrade-check: - name: Check for updates - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - package: [pylint, yamllint, isort, codespell] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - name: Install extra python packages - run: python3 -m pip install --requirement utils/cq/requirements.txt - - name: Check ${{ matrix.package }} version - run: python -m ${{ matrix.package }} --version | tee -a version-pre - - name: Upgrade - run: pip install --upgrade ${{ matrix.package }} - - name: Check ${{ matrix.package }} for version - run: python -m ${{ matrix.package }} --version | diff version-pre - From c86aa7bcc71c10c30bcdeec4ed0c7e996a1544ca Mon Sep 17 00:00:00 2001 From: Liu Xuezhao Date: Tue, 17 Dec 2024 00:15:35 +0800 Subject: [PATCH 03/10] DAOS-16170 cart: refine corpc fail handling for CRT_RPC_FLAG_CO_FAILOUT (#15572) reset rc to 0 to avoid unnecessary RPC_DECREF in fail case in crt_req_send. Signed-off-by: Xuezhao Liu --- src/cart/crt_corpc.c | 17 +++++++++-------- src/cart/crt_rpc.c | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/cart/crt_corpc.c b/src/cart/crt_corpc.c index 5d7b6d79c6e..24a5798a74c 100644 --- a/src/cart/crt_corpc.c +++ b/src/cart/crt_corpc.c @@ -1,5 +1,5 @@ /* - * (C) Copyright 2016-2023 Intel Corporation. + * (C) Copyright 2016-2024 Intel Corporation. * * SPDX-License-Identifier: BSD-2-Clause-Patent */ @@ -777,6 +777,7 @@ crt_corpc_req_hdlr(struct crt_rpc_priv *rpc_priv) struct crt_opc_info *opc_info; struct crt_corpc_ops *co_ops; bool ver_match; + bool co_failout = false; int i, rc = 0; co_info = rpc_priv->crp_corpc_info; @@ -906,18 +907,18 @@ crt_corpc_req_hdlr(struct crt_rpc_priv *rpc_priv) } forward_done: - if (rc != 0 && rpc_priv->crp_flags & CRT_RPC_FLAG_CO_FAILOUT) { - crt_corpc_complete(rpc_priv); - goto out; - } + if (rc != 0 && rpc_priv->crp_flags & CRT_RPC_FLAG_CO_FAILOUT) + co_failout = true; /* NOOP bcast (no child and root excluded) */ - if (co_info->co_child_num == 0 && co_info->co_root_excluded) + if (co_info->co_child_num == 0 && (co_info->co_root_excluded || co_failout)) crt_corpc_complete(rpc_priv); - if (co_info->co_root_excluded == 1) { + if (co_info->co_root_excluded == 1 || co_failout) { if (co_info->co_grp_priv->gp_self == co_info->co_root) { - /* don't return error for root */ + /* don't return error for root to avoid RPC_DECREF in + * fail case in crt_req_send. + */ rc = 0; } D_GOTO(out, rc); diff --git a/src/cart/crt_rpc.c b/src/cart/crt_rpc.c index 575f653d6c8..b9effbeb9ce 100644 --- a/src/cart/crt_rpc.c +++ b/src/cart/crt_rpc.c @@ -1532,7 +1532,7 @@ crt_req_send(crt_rpc_t *req, crt_cb_t complete_cb, void *arg) /* failure already reported through complete cb */ if (complete_cb != NULL) rc = 0; - } else if (!crt_rpc_completed(rpc_priv)) { + } else { RPC_DECREF(rpc_priv); } } From 794f043cff2ca47eb5fae18c265a1a5e61e0eb41 Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Mon, 16 Dec 2024 12:14:53 -0500 Subject: [PATCH 04/10] DAOS-16127 tools: Include relevant pool health in container health (#15489) If the parent pool is in a degraded state, show this in the container health string, as otherwise users may only focus on the container health and miss the other details. Signed-off-by: Michael MacDonald --- src/control/cmd/daos/pretty/health.go | 10 ++++-- src/control/cmd/daos/pretty/health_test.go | 39 ++++++++++++++++++++-- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/src/control/cmd/daos/pretty/health.go b/src/control/cmd/daos/pretty/health.go index 3d3f8ca035d..865f8ade316 100644 --- a/src/control/cmd/daos/pretty/health.go +++ b/src/control/cmd/daos/pretty/health.go @@ -132,12 +132,16 @@ func printPoolHealth(out io.Writer, pi *daos.PoolInfo, verbose bool) { fmt.Fprintf(out, "%s: %s\n", pi.Name(), strings.Join(healthStrings, ",")) } -func printContainerHealth(out io.Writer, ci *daos.ContainerInfo, verbose bool) { +func printContainerHealth(out io.Writer, pi *daos.PoolInfo, ci *daos.ContainerInfo, verbose bool) { if ci == nil { return } - fmt.Fprintf(out, "%s: %s\n", ci.Name(), txtfmt.Title(ci.Health)) + healthStr := txtfmt.Title(ci.Health) + if pi != nil && pi.DisabledTargets > 0 { + healthStr += " (Pool Degraded)" + } + fmt.Fprintf(out, "%s: %s\n", ci.Name(), healthStr) } // PrintSystemHealthInfo pretty-prints the supplied system health struct. @@ -180,7 +184,7 @@ func PrintSystemHealthInfo(out io.Writer, shi *daos.SystemHealthInfo, verbose bo iiiw := txtfmt.NewIndentWriter(iiw) if len(shi.Containers[pool.UUID]) > 0 { for _, cont := range shi.Containers[pool.UUID] { - printContainerHealth(iiiw, cont, verbose) + printContainerHealth(iiiw, pool, cont, verbose) } } else { fmt.Fprintln(iiiw, "No containers in pool.") diff --git a/src/control/cmd/daos/pretty/health_test.go b/src/control/cmd/daos/pretty/health_test.go index 909becb2b0f..19abf0cf9b7 100644 --- a/src/control/cmd/daos/pretty/health_test.go +++ b/src/control/cmd/daos/pretty/health_test.go @@ -291,12 +291,47 @@ var healthyContainer = &daos.ContainerInfo{ func TestPretty_printContainerHealth(t *testing.T) { for name, tc := range map[string]struct { + pi *daos.PoolInfo ci *daos.ContainerInfo verbose bool expPrintStr string }{ "nil ContainerInfo": {}, - "healthy": { + "unhealthy pool, healthy container": { + pi: &daos.PoolInfo{ + DisabledTargets: 1, + }, + ci: healthyContainer, + expPrintStr: fmt.Sprintf(` +%s: Healthy (Pool Degraded) +`, healthyContainer.ContainerLabel), + }, + "unhealthy pool, unhealthy container": { + pi: &daos.PoolInfo{ + DisabledTargets: 1, + }, + ci: func() *daos.ContainerInfo { + clone := *healthyContainer + clone.Health = "UNHEALTHY" + return &clone + }(), + expPrintStr: fmt.Sprintf(` +%s: Unhealthy (Pool Degraded) +`, healthyContainer.ContainerLabel), + }, + "healthy pool, unhealthy container": { + pi: healthyPool, + ci: func() *daos.ContainerInfo { + clone := *healthyContainer + clone.Health = "UNHEALTHY" + return &clone + }(), + expPrintStr: fmt.Sprintf(` +%s: Unhealthy +`, healthyContainer.ContainerLabel), + }, + "healthy pool, healthy container": { + pi: healthyPool, ci: healthyContainer, expPrintStr: fmt.Sprintf(` %s: Healthy @@ -305,7 +340,7 @@ func TestPretty_printContainerHealth(t *testing.T) { } { t.Run(name, func(t *testing.T) { var bld strings.Builder - printContainerHealth(&bld, tc.ci, tc.verbose) + printContainerHealth(&bld, tc.pi, tc.ci, tc.verbose) if diff := cmp.Diff(strings.TrimLeft(tc.expPrintStr, "\n"), bld.String()); diff != "" { t.Fatalf("unexpected pretty-printed string (-want, +got):\n%s\n", diff) From a614b38d8ffa64f762ba76eec2e82ca648e5e3d5 Mon Sep 17 00:00:00 2001 From: Li Wei Date: Tue, 17 Dec 2024 02:15:15 +0900 Subject: [PATCH 05/10] DAOS-16874 tests: Update expected pool_layout_ver (#15610) Update expected pool_layout_ver in dmg_pool_query_test/test_pool_query_basic. Signed-off-by: Li Wei --- src/tests/ftest/control/dmg_pool_query_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/ftest/control/dmg_pool_query_test.py b/src/tests/ftest/control/dmg_pool_query_test.py index 593862a0ea0..d3853c11609 100644 --- a/src/tests/ftest/control/dmg_pool_query_test.py +++ b/src/tests/ftest/control/dmg_pool_query_test.py @@ -85,9 +85,9 @@ def test_pool_query_basic(self): "total": self.params.get("total", path="/run/exp_vals/nvme/*") } ], - "pool_layout_ver": 3, + "pool_layout_ver": 4, "query_mask": self.params.get("query_mask", path="/run/exp_vals/*"), - "upgrade_layout_ver": 3, + "upgrade_layout_ver": 4, "usage": [ { "tier_name": "SCM", From 10053d46fde26c008614b26915b72ad74b16c49f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Dec 2024 14:17:14 -0800 Subject: [PATCH 06/10] Build(deps): Bump phoenix-actions/test-reporting from 10 to 15 (#15617) Bumps [phoenix-actions/test-reporting](https://github.com/phoenix-actions/test-reporting) from 10 to 15. - [Release notes](https://github.com/phoenix-actions/test-reporting/releases) - [Changelog](https://github.com/phoenix-actions/test-reporting/blob/main/CHANGELOG.md) - [Commits](https://github.com/phoenix-actions/test-reporting/compare/v10...v15) --- updated-dependencies: - dependency-name: phoenix-actions/test-reporting dependency-type: direct:production update-type: version-update:semver-major ... Also Pin dependencies per Scorecard Signed-off-by: dependabot[bot] Signed-off-by: Brian J. Murrell --- .github/workflows/rpm-build-and-test-report.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rpm-build-and-test-report.yml b/.github/workflows/rpm-build-and-test-report.yml index d268550d2dc..7b67704b307 100644 --- a/.github/workflows/rpm-build-and-test-report.yml +++ b/.github/workflows/rpm-build-and-test-report.yml @@ -41,7 +41,7 @@ jobs: esac echo "STAGE_NAME=Build RPM on $DISTRO_NAME $DISTRO_VERSION" >> $GITHUB_ENV - name: Test Report - uses: phoenix-actions/test-reporting@v10 + uses: phoenix-actions/test-reporting@f957cd93fc2d848d556fa0d03c57bc79127b6b5e # v15 with: artifact: ${{ env.STAGE_NAME }} test-results name: ${{ env.STAGE_NAME }} Test Results (phoenix-actions) @@ -60,7 +60,7 @@ jobs: - name: Set variables run: echo "STAGE_NAME=Functional Hardware ${{ matrix.stage }}" >> $GITHUB_ENV - name: Test Report - uses: phoenix-actions/test-reporting@v10 + uses: phoenix-actions/test-reporting@f957cd93fc2d848d556fa0d03c57bc79127b6b5e # v15 with: artifact: ${{ env.STAGE_NAME }} test-results name: ${{ env.STAGE_NAME }} Test Results (phoenix-actions) From f68c26c18d6f86c5eb4f5a1ddf5e7f626f9aa9fb Mon Sep 17 00:00:00 2001 From: Niu Yawei Date: Tue, 17 Dec 2024 22:51:39 +0800 Subject: [PATCH 07/10] DAOS-16209 pool: return mem file size on pool query (#15511) * DAOS-16209 pool: return mem file size on pool query Return the mem file size for phase2 pool on pool query. This patch bumped DAOS_POOL_VERSION from 6 to 7, and dropped version 5 support. Signed-off-by: Niu Yawei Signed-off-by: Tom Nabarro Signed-off-by: Tom Nabarro Signed-off-by: Dalton Bohning Co-authored-by: Tom Nabarro Co-authored-by: Tom Nabarro Co-authored-by: Dalton Bohning --- src/control/common/proto/logging.go | 7 +- src/control/server/mgmt_pool.go | 29 +- src/control/server/mgmt_pool_test.go | 108 +++-- src/control/server/util_test.go | 20 +- src/include/daos_srv/pool.h | 13 +- src/include/daos_srv/vos_types.h | 2 + src/mgmt/srv_drpc.c | 30 +- src/mgmt/srv_internal.h | 8 +- src/mgmt/srv_pool.c | 15 +- src/mgmt/tests/mocks.c | 12 +- src/mgmt/tests/mocks.h | 2 + src/mgmt/tests/srv_drpc_tests.c | 13 +- src/pool/cli.c | 10 +- src/pool/rpc.c | 67 +-- src/pool/rpc.h | 611 ++++++++++----------------- src/pool/srv.c | 17 +- src/pool/srv_cli.c | 16 +- src/pool/srv_internal.h | 54 +-- src/pool/srv_pool.c | 223 ++++------ src/pool/srv_target.c | 65 ++- src/vos/vos_layout.h | 4 +- src/vos/vos_pool.c | 5 + src/vos/vos_space.c | 6 + 23 files changed, 611 insertions(+), 726 deletions(-) diff --git a/src/control/common/proto/logging.go b/src/control/common/proto/logging.go index 5de759ce865..7a9677ed2d8 100644 --- a/src/control/common/proto/logging.go +++ b/src/control/common/proto/logging.go @@ -92,7 +92,7 @@ func Debug(msg proto.Message) string { fmt.Fprintf(&bld, "ranks:%s ", ranks.String()) fmt.Fprint(&bld, "tiers:") for i, b := range m.TierBytes { - fmt.Fprintf(&bld, "%d: %d ", i, b) + fmt.Fprintf(&bld, "%d: %s (%d)", i, humanize.Bytes(b), b) if len(m.TierRatio) > i+1 { fmt.Fprintf(&bld, "(%.02f%%) ", m.TierRatio[i]) } @@ -112,9 +112,10 @@ func Debug(msg proto.Message) string { fmt.Fprintf(&bld, "tgt_ranks:%s ", ranks.String()) fmt.Fprint(&bld, "tiers:") for i, b := range m.TierBytes { - fmt.Fprintf(&bld, "%d:%d ", i, b) + fmt.Fprintf(&bld, "%d: %s (%d)", i, humanize.Bytes(b), b) } - fmt.Fprintf(&bld, "meta-file-size:%s", humanize.Bytes(m.MemFileBytes)) + fmt.Fprintf(&bld, "meta-file-size: %s (%d)", humanize.Bytes(m.MemFileBytes), + m.MemFileBytes) case *mgmtpb.PoolEvictReq: fmt.Fprintf(&bld, "%T pool:%s", m, m.Id) if len(m.Handles) > 0 { diff --git a/src/control/server/mgmt_pool.go b/src/control/server/mgmt_pool.go index ef30d975bb8..1e0b8ac33d0 100644 --- a/src/control/server/mgmt_pool.go +++ b/src/control/server/mgmt_pool.go @@ -482,6 +482,14 @@ func (svc *mgmtSvc) poolCreate(parent context.Context, req *mgmtpb.PoolCreateReq return nil, errors.Wrap(err, "unmarshal PoolCreate response") } + // Zero mem_file_bytes in non-MD-on-SSD mode. + if !svc.harness.Instances()[0].GetStorage().BdevRoleMetaConfigured() { + resp.MemFileBytes = 0 + } else { + svc.log.Tracef("%T mem_file_bytes: %s (%d)", resp, + humanize.Bytes(resp.MemFileBytes), resp.MemFileBytes) + } + if resp.GetStatus() != 0 { if err := svc.sysdb.RemovePoolService(ctx, ps.PoolUUID); err != nil { return nil, err @@ -960,11 +968,12 @@ func (svc *mgmtSvc) PoolQuery(ctx context.Context, req *mgmtpb.PoolQueryReq) (*m // Preserve compatibility with pre-2.6 callers. resp.Leader = resp.SvcLdr - // TODO DAOS-16209: After VOS query API is updated, zero-value mem_file_bytes will be - // returned in non-MD-on-SSD mode and this hack can be removed. - storage := svc.harness.Instances()[0].GetStorage() - if !storage.ControlMetadataPathConfigured() { + // Zero mem_file_bytes in non-MD-on-SSD mode. + if !svc.harness.Instances()[0].GetStorage().BdevRoleMetaConfigured() { resp.MemFileBytes = 0 + } else { + svc.log.Tracef("%T mem_file_bytes: %s (%d)", resp, + humanize.Bytes(resp.MemFileBytes), resp.MemFileBytes) } return resp, nil @@ -986,13 +995,17 @@ func (svc *mgmtSvc) PoolQueryTarget(ctx context.Context, req *mgmtpb.PoolQueryTa return nil, errors.Wrap(err, "unmarshal PoolQueryTarget response") } - // TODO DAOS-16209: After VOS query API is updated, zero-value mem_file_bytes will be - // returned in non-MD-on-SSD mode and this hack can be removed. - storage := svc.harness.Instances()[0].GetStorage() - if !storage.ControlMetadataPathConfigured() { + // Zero mem_file_bytes in non-MD-on-SSD mode. + if !svc.harness.Instances()[0].GetStorage().BdevRoleMetaConfigured() { for _, tgtInfo := range resp.Infos { tgtInfo.MemFileBytes = 0 } + } else { + for _, tgtInfo := range resp.Infos { + svc.log.Tracef("%T mem_file_bytes: %s (%d)", resp, + humanize.Bytes(tgtInfo.MemFileBytes), tgtInfo.MemFileBytes) + break + } } return resp, nil diff --git a/src/control/server/mgmt_pool_test.go b/src/control/server/mgmt_pool_test.go index 25b45d07567..d5a056b8938 100644 --- a/src/control/server/mgmt_pool_test.go +++ b/src/control/server/mgmt_pool_test.go @@ -24,7 +24,6 @@ import ( mgmtpb "github.com/daos-stack/daos/src/control/common/proto/mgmt" "github.com/daos-stack/daos/src/control/common/test" "github.com/daos-stack/daos/src/control/drpc" - "github.com/daos-stack/daos/src/control/events" "github.com/daos-stack/daos/src/control/lib/daos" "github.com/daos-stack/daos/src/control/lib/ranklist" "github.com/daos-stack/daos/src/control/logging" @@ -504,6 +503,16 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { TgtRanks: []uint32{0, 1}, }, }, + "create with memory file ratio; mdonssd not enabled": { + targetCount: 8, + req: &mgmtpb.PoolCreateReq{ + Uuid: test.MockUUID(1), + TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, + MemRatio: storage.DefaultMemoryFileRatio, + Properties: testPoolLabelProp(), + }, + expErr: errors.New("MD-on-SSD has not been enabled"), + }, "successful creation with memory file ratio": { targetCount: 8, mdonssdEnabled: true, @@ -524,6 +533,23 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { TgtRanks: []uint32{0, 1}, }, }, + "successful creation with memory file bytes in resp; mdonssd not enabled": { + targetCount: 8, + req: &mgmtpb.PoolCreateReq{ + Uuid: test.MockUUID(1), + TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, + Properties: testPoolLabelProp(), + }, + drpcRet: &mgmtpb.PoolCreateResp{ + TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, + MemFileBytes: 100 * humanize.GiByte, + TgtRanks: []uint32{0, 1}, + }, + expResp: &mgmtpb.PoolCreateResp{ + TierBytes: []uint64{100 * humanize.GiByte, 10 * humanize.TByte}, + TgtRanks: []uint32{0, 1}, + }, + }, "successful creation minimum size": { targetCount: 8, req: &mgmtpb.PoolCreateReq{ @@ -629,8 +655,6 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { buf.Reset() defer test.ShowBufferOnFailure(t, buf) - ctx := test.Context(t) - if tc.mgmtSvc == nil { tier := storage.NewTierConfig(). WithStorageClass("nvme"). @@ -639,30 +663,11 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { tier.WithBdevDeviceRoles(7) } engineCfg := engine.MockConfig(). - WithTargetCount(tc.targetCount). + WithTargetCount(16). WithStorage(tier) - r := engine.NewTestRunner(nil, engineCfg) - if _, err := r.Start(ctx); err != nil { - t.Fatal(err) - } - mp := storage.NewProvider(log, 0, &engineCfg.Storage, nil, nil, nil, nil) - srv := NewEngineInstance(log, mp, nil, r, nil) - srv.setDrpcSocket("/dontcare") - srv.ready.SetTrue() - - harness := NewEngineHarness(log) - if err := harness.AddInstance(srv); err != nil { - panic(err) - } - harness.started.SetTrue() - - db := raft.MockDatabase(t, log) - ms := system.MockMembership(t, log, db, mockTCPResolver) - tc.mgmtSvc = newMgmtSvc(harness, ms, db, nil, - events.NewPubSub(ctx, log)) - tc.mgmtSvc.startAsyncLoops(ctx) + tc.mgmtSvc = newTestMgmtSvcWithProvider(t, log, mp) } numMembers := tc.memberCount @@ -2315,11 +2320,12 @@ func TestServer_MgmtSvc_PoolQuery(t *testing.T) { } for name, tc := range map[string]struct { - mgmtSvc *mgmtSvc - setupMockDrpc func(_ *mgmtSvc, _ error) - req *mgmtpb.PoolQueryReq - expResp *mgmtpb.PoolQueryResp - expErr error + mdonssdEnabled bool + mgmtSvc *mgmtSvc + setupMockDrpc func(_ *mgmtSvc, _ error) + req *mgmtpb.PoolQueryReq + expResp *mgmtpb.PoolQueryResp + expErr error }{ "nil request": { expErr: errors.New("nil request"), @@ -2369,15 +2375,16 @@ func TestServer_MgmtSvc_PoolQuery(t *testing.T) { Uuid: mockUUID, }, }, - "successful query (includes pre-2.6 Leader field)": { + "successful query (includes pre-2.6 Leader field); mdonssd not enabled": { req: &mgmtpb.PoolQueryReq{ Id: mockUUID, }, setupMockDrpc: func(svc *mgmtSvc, err error) { resp := &mgmtpb.PoolQueryResp{ - State: mgmtpb.PoolServiceState_Ready, - Uuid: mockUUID, - SvcLdr: 42, + State: mgmtpb.PoolServiceState_Ready, + Uuid: mockUUID, + SvcLdr: 42, + MemFileBytes: humanize.GiByte, } setupMockDrpcClient(svc, resp, nil) }, @@ -2388,19 +2395,50 @@ func TestServer_MgmtSvc_PoolQuery(t *testing.T) { Leader: 42, }, }, + "successful query; mdonssd enabled": { + mdonssdEnabled: true, + req: &mgmtpb.PoolQueryReq{ + Id: mockUUID, + }, + setupMockDrpc: func(svc *mgmtSvc, err error) { + resp := &mgmtpb.PoolQueryResp{ + State: mgmtpb.PoolServiceState_Ready, + Uuid: mockUUID, + MemFileBytes: humanize.GiByte, + } + setupMockDrpcClient(svc, resp, nil) + }, + expResp: &mgmtpb.PoolQueryResp{ + State: mgmtpb.PoolServiceState_Ready, + Uuid: mockUUID, + MemFileBytes: humanize.GiByte, + }, + }, } { t.Run(name, func(t *testing.T) { buf.Reset() defer test.ShowBufferOnFailure(t, buf) if tc.mgmtSvc == nil { - tc.mgmtSvc = newTestMgmtSvc(t, log) + tier := storage.NewTierConfig(). + WithStorageClass("nvme"). + WithBdevDeviceList("foo", "bar") + if tc.mdonssdEnabled { + tier.WithBdevDeviceRoles(7) + } + engineCfg := engine.MockConfig(). + WithTargetCount(16). + WithStorage(tier) + mp := storage.NewProvider(log, 0, &engineCfg.Storage, + nil, nil, nil, nil) + tc.mgmtSvc = newTestMgmtSvcWithProvider(t, log, mp) } addTestPools(t, tc.mgmtSvc.sysdb, mockUUID) if tc.setupMockDrpc == nil { tc.setupMockDrpc = func(svc *mgmtSvc, err error) { - setupSvcDrpcClient(svc, 0, getMockDrpcClient(tc.expResp, tc.expErr)) + setupSvcDrpcClient(svc, 0, + getMockDrpcClient(tc.expResp, tc.expErr)) } } tc.setupMockDrpc(tc.mgmtSvc, tc.expErr) diff --git a/src/control/server/util_test.go b/src/control/server/util_test.go index ac6b42ed51c..67542c47d3a 100644 --- a/src/control/server/util_test.go +++ b/src/control/server/util_test.go @@ -234,15 +234,13 @@ func mockTCPResolver(netString string, address string) (*net.TCPAddr, error) { return &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 10001}, nil } -// newTestMgmtSvc creates a mgmtSvc that contains an EngineInstance -// properly set up as an MS. -func newTestMgmtSvc(t *testing.T, log logging.Logger) *mgmtSvc { +// newTestMgmtSvcWithProvider creates a mgmtSvc that contains an EngineInstance +// properly set up as an MS using an input storage provider. +func newTestMgmtSvcWithProvider(t *testing.T, log logging.Logger, provider *storage.Provider) *mgmtSvc { harness := NewEngineHarness(log) - provider := storage.MockProvider(log, 0, nil, nil, nil, nil, nil) - - srv := newTestEngine(log, true, provider) - if err := harness.AddInstance(srv); err != nil { + e := newTestEngine(log, true, provider) + if err := harness.AddInstance(e); err != nil { t.Fatal(err) } harness.started.SetTrue() @@ -254,9 +252,17 @@ func newTestMgmtSvc(t *testing.T, log logging.Logger) *mgmtSvc { svc.batchInterval = 100 * time.Microsecond // Speed up tests svc.startAsyncLoops(ctx) svc.startLeaderLoops(ctx) + return svc } +// newTestMgmtSvc creates a mgmtSvc that contains an EngineInstance +// properly set up as an MS. +func newTestMgmtSvc(t *testing.T, log logging.Logger) *mgmtSvc { + return newTestMgmtSvcWithProvider(t, log, + storage.MockProvider(log, 0, nil, nil, nil, nil, nil)) +} + // newTestMgmtSvcMulti creates a mgmtSvc that contains the requested // number of EngineInstances. If requested, the first instance is // configured as an access point. diff --git a/src/include/daos_srv/pool.h b/src/include/daos_srv/pool.h index 9e2ca08c271..abb539fa74b 100644 --- a/src/include/daos_srv/pool.h +++ b/src/include/daos_srv/pool.h @@ -306,13 +306,14 @@ int dsc_pool_svc_delete_acl(uuid_t pool_uuid, d_rank_list_t *ranks, uint64_t dea enum daos_acl_principal_type principal_type, const char *principal_name); -int - dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, - d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ranks, - d_rank_list_t **dead_ranks, daos_pool_info_t *pool_info, - uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver); +int dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, + d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ranks, + d_rank_list_t **dead_ranks, daos_pool_info_t *pool_info, + uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver, + uint64_t *mem_file_bytes); int dsc_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, - d_rank_t rank, uint32_t tgt_idx, daos_target_info_t *ti); + d_rank_t rank, uint32_t tgt_idx, daos_target_info_t *ti, + uint64_t *mem_file_bytes); int ds_pool_prop_fetch(struct ds_pool *pool, unsigned int bit, daos_prop_t **prop_out); diff --git a/src/include/daos_srv/vos_types.h b/src/include/daos_srv/vos_types.h index 0a52851c390..a0178ca52db 100644 --- a/src/include/daos_srv/vos_types.h +++ b/src/include/daos_srv/vos_types.h @@ -130,6 +130,8 @@ struct vos_pool_space { /** Total & free non-evictable space for md-on-ssd phase2 pool */ uint64_t vps_ne_total; uint64_t vps_ne_free; + /* Memory file size for md-on-ssd pool */ + uint64_t vps_mem_bytes; }; #define SCM_TOTAL(vps) ((vps)->vps_space.s_total[DAOS_MEDIA_SCM]) diff --git a/src/mgmt/srv_drpc.c b/src/mgmt/srv_drpc.c index fb9712bbe8b..0689d3f8441 100644 --- a/src/mgmt/srv_drpc.c +++ b/src/mgmt/srv_drpc.c @@ -382,6 +382,7 @@ static int pool_create_fill_resp(Mgmt__PoolCreateResp *resp, uuid_t uuid, d_rank int index; d_rank_list_t *enabled_ranks = NULL; daos_pool_info_t pool_info = { .pi_bits = DPI_ENGINES_ENABLED | DPI_SPACE }; + uint64_t mem_file_bytes = 0; D_ASSERT(svc_ranks != NULL); D_ASSERT(svc_ranks->rl_nr > 0); @@ -394,8 +395,8 @@ static int pool_create_fill_resp(Mgmt__PoolCreateResp *resp, uuid_t uuid, d_rank D_DEBUG(DB_MGMT, "%d service replicas\n", svc_ranks->rl_nr); - rc = - ds_mgmt_pool_query(uuid, svc_ranks, &enabled_ranks, NULL, NULL, &pool_info, NULL, NULL); + rc = ds_mgmt_pool_query(uuid, svc_ranks, &enabled_ranks, NULL, NULL, &pool_info, NULL, NULL, + &mem_file_bytes); if (rc != 0) { D_ERROR("Failed to query created pool: rc=%d\n", rc); D_GOTO(out, rc); @@ -409,9 +410,11 @@ static int pool_create_fill_resp(Mgmt__PoolCreateResp *resp, uuid_t uuid, d_rank D_GOTO(out, rc); } + D_ASSERT(resp->n_tgt_ranks > 0); for (index = 0; index < DAOS_MEDIA_MAX; ++index) { D_ASSERT(pool_info.pi_space.ps_space.s_total[index] % resp->n_tgt_ranks == 0); } + D_ASSERT(mem_file_bytes % resp->n_tgt_ranks == 0); D_ALLOC_ARRAY(resp->tier_bytes, DAOS_MEDIA_MAX); if (resp->tier_bytes == NULL) { rc = -DER_NOMEM; @@ -422,6 +425,7 @@ static int pool_create_fill_resp(Mgmt__PoolCreateResp *resp, uuid_t uuid, d_rank resp->tier_bytes[index] = pool_info.pi_space.ps_space.s_total[index] / resp->n_tgt_ranks; } + resp->mem_file_bytes = mem_file_bytes / resp->n_tgt_ranks; out: d_rank_list_free(enabled_ranks); @@ -519,14 +523,6 @@ ds_mgmt_drpc_pool_create(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) rc = pool_create_fill_resp(&resp, pool_uuid, svc); d_rank_list_free(svc); - /** - * TODO DAOS-16209: Populate per-rank VOS-file sizes. For now just calculate here based on - * the supplied input values but really should be returned from - * ds_mgmt_pool_query() through the VOS query API and set in - * pool_create_fill_resp(). Return zero for non-MD-on-SSD mode. - */ - resp.mem_file_bytes = req->tier_bytes[DAOS_MEDIA_SCM] * req->mem_ratio; - out: resp.status = rc; len = mgmt__pool_create_resp__get_packed_size(&resp); @@ -1797,7 +1793,8 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) pool_info.pi_bits = req->query_mask; rc = ds_mgmt_pool_query(uuid, svc_ranks, &enabled_ranks, &disabled_ranks, &dead_ranks, - &pool_info, &resp.pool_layout_ver, &resp.upgrade_layout_ver); + &pool_info, &resp.pool_layout_ver, &resp.upgrade_layout_ver, + &resp.mem_file_bytes); if (rc != 0) { DL_ERROR(rc, DF_UUID ": Failed to query the pool", DP_UUID(uuid)); D_GOTO(error, rc); @@ -1866,13 +1863,6 @@ ds_mgmt_drpc_pool_query(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) pool_rebuild_status_from_info(&rebuild, &pool_info.pi_rebuild_st); resp.rebuild = &rebuild; - /** - * TODO DAOS-16209: Populate VOS-file sizes in response. For now just return the meta-blob - * size until VOS query API is updated. When updated, zero-value should - * be returned in non-MD-on-SSD mode. - */ - resp.mem_file_bytes = scm.total; - error: resp.status = rc; @@ -1913,6 +1903,7 @@ ds_mgmt_drpc_pool_query_targets(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) uint8_t *body; daos_target_info_t *infos = NULL; Mgmt__PoolQueryTargetInfo *resp_infos = NULL; + uint64_t mem_file_bytes = 0; req = mgmt__pool_query_target_req__unpack(&alloc.alloc, drpc_req->body.len, drpc_req->body.data); @@ -1938,7 +1929,7 @@ ds_mgmt_drpc_pool_query_targets(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) if (tgts == NULL) D_GOTO(out_ranks, rc = -DER_NOMEM); - rc = ds_mgmt_pool_query_targets(uuid, svc_ranks, req->rank, tgts, &infos); + rc = ds_mgmt_pool_query_targets(uuid, svc_ranks, req->rank, tgts, &infos, &mem_file_bytes); if (rc != 0) { D_ERROR("ds_mgmt_pool_query_targets() failed, pool %s rank %u, "DF_RC"\n", req->id, req->rank, DP_RC(rc)); @@ -1983,6 +1974,7 @@ ds_mgmt_drpc_pool_query_targets(Drpc__Call *drpc_req, Drpc__Response *drpc_resp) resp.infos[i]->space[j]->free = infos[i].ta_space.s_free[j]; resp.infos[i]->space[j]->media_type = j; } + resp.infos[i]->mem_file_bytes = mem_file_bytes; } out_infos: diff --git a/src/mgmt/srv_internal.h b/src/mgmt/srv_internal.h index 736c5e1878e..473c9aa6f29 100644 --- a/src/mgmt/srv_internal.h +++ b/src/mgmt/srv_internal.h @@ -123,9 +123,11 @@ int ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ranks, d_rank_list_t **dead_ranks, daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, - uint32_t *upgrade_layout_ver); -int ds_mgmt_pool_query_targets(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_t rank, - d_rank_list_t *tgts, daos_target_info_t **infos); + uint32_t *upgrade_layout_ver, uint64_t *mem_file_bytes); +int + ds_mgmt_pool_query_targets(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_t rank, + d_rank_list_t *tgts, daos_target_info_t **infos, + uint64_t *mem_file_bytes); int ds_mgmt_cont_set_owner(uuid_t pool_uuid, d_rank_list_t *svc_ranks, const char *cont_id, diff --git a/src/mgmt/srv_pool.c b/src/mgmt/srv_pool.c index 6059c20de73..14bab9651b2 100644 --- a/src/mgmt/srv_pool.c +++ b/src/mgmt/srv_pool.c @@ -402,7 +402,7 @@ int ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ranks, d_rank_list_t **dead_ranks, daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, - uint32_t *upgrade_layout_ver) + uint32_t *upgrade_layout_ver, uint64_t *mem_file_bytes) { if (pool_info == NULL) { D_ERROR("pool_info was NULL\n"); @@ -413,7 +413,7 @@ ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **e return dsc_pool_svc_query(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), enabled_ranks, disabled_ranks, dead_ranks, pool_info, pool_layout_ver, - upgrade_layout_ver); + upgrade_layout_ver, mem_file_bytes); } /** @@ -432,7 +432,8 @@ ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **e */ int ds_mgmt_pool_query_targets(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_t rank, - d_rank_list_t *tgts, daos_target_info_t **infos) + d_rank_list_t *tgts, daos_target_info_t **infos, + uint64_t *mem_file_bytes) { int rc = 0; uint32_t i; @@ -448,15 +449,21 @@ ds_mgmt_pool_query_targets(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_t D_GOTO(out, rc = -DER_NOMEM); for (i = 0; i < tgts->rl_nr; i++) { + uint64_t mem_bytes = 0; + D_DEBUG(DB_MGMT, "Querying pool "DF_UUID" rank %u tgt %u\n", DP_UUID(pool_uuid), rank, tgts->rl_ranks[i]); rc = dsc_pool_svc_query_target(pool_uuid, svc_ranks, mgmt_ps_call_deadline(), rank, - tgts->rl_ranks[i], &out_infos[i]); + tgts->rl_ranks[i], &out_infos[i], &mem_bytes); if (rc != 0) { D_ERROR(DF_UUID": dsc_pool_svc_query_target() failed rank %u tgt %u\n", DP_UUID(pool_uuid), rank, tgts->rl_ranks[i]); goto out; } + if (mem_file_bytes) { + D_ASSERT(i == 0 || *mem_file_bytes == mem_bytes); + *mem_file_bytes = mem_bytes; + } } out: diff --git a/src/mgmt/tests/mocks.c b/src/mgmt/tests/mocks.c index cc597451225..e266a114572 100644 --- a/src/mgmt/tests/mocks.c +++ b/src/mgmt/tests/mocks.c @@ -277,6 +277,7 @@ void mock_ds_mgmt_pool_list_cont_teardown(void) int ds_mgmt_pool_query_return; uuid_t ds_mgmt_pool_query_uuid; daos_pool_info_t ds_mgmt_pool_query_info_out; +uint64_t ds_mgmt_pool_query_mem_bytes; daos_pool_info_t ds_mgmt_pool_query_info_in; void *ds_mgmt_pool_query_info_ptr; d_rank_list_t *ds_mgmt_pool_query_enabled_ranks_out; @@ -287,7 +288,7 @@ int ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ranks, d_rank_list_t **dead_ranks, daos_pool_info_t *pool_info, uint32_t *pool_layout_ver, - uint32_t *upgrade_layout_ver) + uint32_t *upgrade_layout_ver, uint64_t *mem_file_bytes) { /* If function is to return with an error, pool_info and ranks will not be filled. */ if (ds_mgmt_pool_query_return != 0) @@ -321,6 +322,7 @@ ds_mgmt_pool_query(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_list_t **e ds_mgmt_pool_query_info_in = *pool_info; *pool_info = ds_mgmt_pool_query_info_out; + *mem_file_bytes = ds_mgmt_pool_query_mem_bytes; return ds_mgmt_pool_query_return; /* 0 */ } @@ -340,10 +342,12 @@ mock_ds_mgmt_pool_query_setup(void) int ds_mgmt_pool_query_targets_return; uuid_t ds_mgmt_pool_query_targets_uuid; daos_target_info_t *ds_mgmt_pool_query_targets_info_out; +uint64_t ds_mgmt_pool_query_targets_mem_bytes; int ds_mgmt_pool_query_targets(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_t rank, - d_rank_list_t *tgts, daos_target_info_t **infos) + d_rank_list_t *tgts, daos_target_info_t **infos, + uint64_t *mem_file_bytes) { /* If function is to return with an error, infos will not be filled. */ if (ds_mgmt_pool_query_targets_return != 0) @@ -355,6 +359,8 @@ ds_mgmt_pool_query_targets(uuid_t pool_uuid, d_rank_list_t *svc_ranks, d_rank_t memcpy(*infos, ds_mgmt_pool_query_targets_info_out, tgts->rl_nr * sizeof(daos_target_info_t)); } + if (mem_file_bytes != NULL) + *mem_file_bytes = ds_mgmt_pool_query_targets_mem_bytes; return ds_mgmt_pool_query_targets_return; /* 0 */ } @@ -375,6 +381,7 @@ mock_ds_mgmt_pool_query_targets_gen_infos(uint32_t n_infos) infos[i].ta_space.s_free[DAOS_MEDIA_NVME] = 600000000 + i; } ds_mgmt_pool_query_targets_info_out = infos; + ds_mgmt_pool_query_targets_mem_bytes = 2000000000; } void @@ -383,6 +390,7 @@ mock_ds_mgmt_pool_query_targets_setup(void) ds_mgmt_pool_query_targets_return = 0; uuid_clear(ds_mgmt_pool_query_targets_uuid); ds_mgmt_pool_query_targets_info_out = NULL; + ds_mgmt_pool_query_targets_mem_bytes = 0; } void diff --git a/src/mgmt/tests/mocks.h b/src/mgmt/tests/mocks.h index ea93e7199b1..b4e2e5947be 100644 --- a/src/mgmt/tests/mocks.h +++ b/src/mgmt/tests/mocks.h @@ -107,6 +107,7 @@ extern int ds_mgmt_pool_query_return; extern uuid_t ds_mgmt_pool_query_uuid; extern daos_pool_info_t ds_mgmt_pool_query_info_in; extern daos_pool_info_t ds_mgmt_pool_query_info_out; +extern uint64_t ds_mgmt_pool_query_mem_bytes; extern void *ds_mgmt_pool_query_info_ptr; extern d_rank_list_t *ds_mgmt_pool_query_enabled_ranks_out; extern d_rank_list_t *ds_mgmt_pool_query_disabled_ranks_out; @@ -120,6 +121,7 @@ void mock_ds_mgmt_pool_query_setup(void); extern int ds_mgmt_pool_query_targets_return; extern uuid_t ds_mgmt_pool_query_targets_uuid; extern daos_target_info_t *ds_mgmt_pool_query_targets_info_out; +extern uint64_t ds_mgmt_pool_query_targets_mem_bytes; void mock_ds_mgmt_pool_query_targets_setup(void); void mock_ds_mgmt_pool_query_targets_teardown(void); diff --git a/src/mgmt/tests/srv_drpc_tests.c b/src/mgmt/tests/srv_drpc_tests.c index 7c7c5e26c70..8c7197a38a0 100644 --- a/src/mgmt/tests/srv_drpc_tests.c +++ b/src/mgmt/tests/srv_drpc_tests.c @@ -1394,6 +1394,8 @@ expect_query_resp_with_info(daos_pool_info_t *exp_info, expect_storage_usage(&exp_info->pi_space, DAOS_MEDIA_NVME, pq_resp->tier_stats[DAOS_MEDIA_NVME]); + assert_int_equal(pq_resp->mem_file_bytes, ds_mgmt_pool_query_mem_bytes); + assert_non_null(pq_resp->rebuild); expect_rebuild_status(&exp_info->pi_rebuild_st, exp_state, pq_resp->rebuild); @@ -1413,6 +1415,7 @@ test_drpc_pool_query_success(void **state) init_test_pool_info(&exp_info); init_test_rebuild_status(&exp_info.pi_rebuild_st); ds_mgmt_pool_query_info_out = exp_info; + ds_mgmt_pool_query_mem_bytes = 11; setup_pool_query_drpc_call(&call, TEST_UUID, flags); @@ -1448,6 +1451,7 @@ test_drpc_pool_query_success_rebuild_busy(void **state) init_test_rebuild_status(&exp_info.pi_rebuild_st); exp_info.pi_rebuild_st.rs_version = 1; ds_mgmt_pool_query_info_out = exp_info; + ds_mgmt_pool_query_mem_bytes = 11; setup_pool_query_drpc_call(&call, TEST_UUID, 0); @@ -1473,6 +1477,7 @@ test_drpc_pool_query_success_rebuild_done(void **state) exp_info.pi_rebuild_st.rs_version = 1; exp_info.pi_rebuild_st.rs_state = DRS_COMPLETED; ds_mgmt_pool_query_info_out = exp_info; + ds_mgmt_pool_query_mem_bytes = 11; setup_pool_query_drpc_call(&call, TEST_UUID, 0); @@ -1498,6 +1503,7 @@ test_drpc_pool_query_success_rebuild_err(void **state) exp_info.pi_rebuild_st.rs_errno = -DER_MISC; ds_mgmt_pool_query_info_out = exp_info; + ds_mgmt_pool_query_mem_bytes = 11; /* * rebuild results returned to us shouldn't include the number of * objects/records if there's an error. @@ -1583,7 +1589,8 @@ expect_drpc_pool_query_targets_resp_with_error(Drpc__Response *resp, int expecte static void expect_drpc_pool_query_targets_resp_with_targets(Drpc__Response *resp, daos_target_info_t *infos, - uint32_t exp_infos_len) + uint32_t exp_infos_len, + uint64_t mem_file_bytes) { Mgmt__PoolQueryTargetResp *pqt_resp = NULL; uint32_t i; @@ -1604,6 +1611,7 @@ expect_drpc_pool_query_targets_resp_with_targets(Drpc__Response *resp, assert_int_equal(pqt_resp->infos[i]->type, infos[i].ta_type); assert_int_equal(pqt_resp->infos[i]->state, infos[i].ta_state); assert_int_equal(pqt_resp->infos[i]->n_space, DAOS_MEDIA_MAX); + assert_int_equal(pqt_resp->infos[i]->mem_file_bytes, mem_file_bytes); for (j = 0; j < DAOS_MEDIA_MAX; j++) { Mgmt__StorageTargetUsage *space = pqt_resp->infos[i]->space[j]; @@ -1666,7 +1674,8 @@ test_drpc_pool_query_targets_with_targets(void **state) expect_drpc_pool_query_targets_resp_with_targets(&resp, ds_mgmt_pool_query_targets_info_out, - n_tgts); + n_tgts, + ds_mgmt_pool_query_targets_mem_bytes); D_FREE(call.body.data); D_FREE(resp.body.data); diff --git a/src/pool/cli.c b/src/pool/cli.c index 6825f57568f..94e26e47f5d 100644 --- a/src/pool/cli.c +++ b/src/pool/cli.c @@ -205,15 +205,15 @@ dc_pool_init(void) daos_register_key(&dc_pool_module_key); dc_pool_proto_version = 0; - rc = daos_rpc_proto_query(pool_proto_fmt_v5.cpf_base, ver_array, 2, &dc_pool_proto_version); + rc = daos_rpc_proto_query(pool_proto_fmt_v6.cpf_base, ver_array, 2, &dc_pool_proto_version); if (rc) return rc; if (dc_pool_proto_version == DAOS_POOL_VERSION - 1) { - rc = daos_rpc_register(&pool_proto_fmt_v5, POOL_PROTO_CLI_COUNT, NULL, + rc = daos_rpc_register(&pool_proto_fmt_v6, POOL_PROTO_CLI_COUNT, NULL, DAOS_POOL_MODULE); } else if (dc_pool_proto_version == DAOS_POOL_VERSION) { - rc = daos_rpc_register(&pool_proto_fmt_v6, POOL_PROTO_CLI_COUNT, NULL, + rc = daos_rpc_register(&pool_proto_fmt_v7, POOL_PROTO_CLI_COUNT, NULL, DAOS_POOL_MODULE); } else { D_ERROR("%d version pool RPC not supported.\n", dc_pool_proto_version); @@ -236,9 +236,9 @@ dc_pool_fini(void) int rc; if (dc_pool_proto_version == DAOS_POOL_VERSION - 1) { - rc = daos_rpc_unregister(&pool_proto_fmt_v5); - } else if (dc_pool_proto_version == DAOS_POOL_VERSION) { rc = daos_rpc_unregister(&pool_proto_fmt_v6); + } else if (dc_pool_proto_version == DAOS_POOL_VERSION) { + rc = daos_rpc_unregister(&pool_proto_fmt_v7); } else { rc = -DER_PROTO; DL_ERROR(rc, "%d version pool RPC not supported", dc_pool_proto_version); diff --git a/src/pool/rpc.c b/src/pool/rpc.c index bcf1fbdd75e..4d297302399 100644 --- a/src/pool/rpc.c +++ b/src/pool/rpc.c @@ -65,7 +65,6 @@ crt_proc_struct_rsvc_hint(crt_proc_t proc, crt_proc_op_t proc_op, } CRT_RPC_DEFINE(pool_op, DAOS_ISEQ_POOL_OP, DAOS_OSEQ_POOL_OP) -CRT_RPC_DEFINE(pool_op_v6, DAOS_ISEQ_POOL_OP_V6, DAOS_OSEQ_POOL_OP) static int crt_proc_struct_pool_op_in(crt_proc_t proc, crt_proc_op_t proc_op, @@ -81,60 +80,40 @@ crt_proc_struct_pool_op_out(crt_proc_t proc, crt_proc_op_t proc_op, return crt_proc_pool_op_out(proc, data); } -static int -crt_proc_struct_pool_op_v6_in(crt_proc_t proc, crt_proc_op_t proc_op, struct pool_op_v6_in *data) -{ - return crt_proc_pool_op_v6_in(proc, data); -} - CRT_RPC_DEFINE(pool_create, DAOS_ISEQ_POOL_CREATE, DAOS_OSEQ_POOL_CREATE) -CRT_RPC_DEFINE(pool_connect_v6, DAOS_ISEQ_POOL_CONNECT_V6, DAOS_OSEQ_POOL_CONNECT) CRT_RPC_DEFINE(pool_connect, DAOS_ISEQ_POOL_CONNECT, DAOS_OSEQ_POOL_CONNECT) -CRT_RPC_DEFINE(pool_disconnect_v6, DAOS_ISEQ_POOL_DISCONNECT_V6, DAOS_OSEQ_POOL_DISCONNECT) CRT_RPC_DEFINE(pool_disconnect, DAOS_ISEQ_POOL_DISCONNECT, DAOS_OSEQ_POOL_DISCONNECT) -CRT_RPC_DEFINE(pool_query_v6, DAOS_ISEQ_POOL_QUERY_V6, DAOS_OSEQ_POOL_QUERY) +CRT_RPC_DEFINE(pool_query_v6, DAOS_ISEQ_POOL_QUERY, DAOS_OSEQ_POOL_QUERY_V6) CRT_RPC_DEFINE(pool_query, DAOS_ISEQ_POOL_QUERY, DAOS_OSEQ_POOL_QUERY) -CRT_RPC_DEFINE(pool_attr_list_v6, DAOS_ISEQ_POOL_ATTR_LIST_V6, DAOS_OSEQ_POOL_ATTR_LIST) CRT_RPC_DEFINE(pool_attr_list, DAOS_ISEQ_POOL_ATTR_LIST, DAOS_OSEQ_POOL_ATTR_LIST) -CRT_RPC_DEFINE(pool_attr_get_v6, DAOS_ISEQ_POOL_ATTR_GET_V6, DAOS_OSEQ_POOL_OP) CRT_RPC_DEFINE(pool_attr_get, DAOS_ISEQ_POOL_ATTR_GET, DAOS_OSEQ_POOL_OP) -CRT_RPC_DEFINE(pool_attr_set_v6, DAOS_ISEQ_POOL_ATTR_SET_V6, DAOS_OSEQ_POOL_OP) CRT_RPC_DEFINE(pool_attr_set, DAOS_ISEQ_POOL_ATTR_SET, DAOS_OSEQ_POOL_OP) -CRT_RPC_DEFINE(pool_attr_del_v6, DAOS_ISEQ_POOL_ATTR_DEL_V6, DAOS_OSEQ_POOL_OP) CRT_RPC_DEFINE(pool_attr_del, DAOS_ISEQ_POOL_ATTR_DEL, DAOS_OSEQ_POOL_OP) CRT_RPC_DEFINE(pool_replicas_add, DAOS_ISEQ_POOL_MEMBERSHIP, DAOS_OSEQ_POOL_MEMBERSHIP) CRT_RPC_DEFINE(pool_replicas_remove, DAOS_ISEQ_POOL_MEMBERSHIP, DAOS_OSEQ_POOL_MEMBERSHIP) -CRT_RPC_DEFINE(pool_extend, DAOS_ISEQ_POOL_EXTEND_V6, DAOS_OSEQ_POOL_EXTEND) -CRT_RPC_DEFINE(pool_add_v6, DAOS_ISEQ_POOL_TGT_UPDATE_V6, DAOS_OSEQ_POOL_TGT_UPDATE) +CRT_RPC_DEFINE(pool_extend, DAOS_ISEQ_POOL_EXTEND, DAOS_OSEQ_POOL_EXTEND) CRT_RPC_DEFINE(pool_add, DAOS_ISEQ_POOL_TGT_UPDATE, DAOS_OSEQ_POOL_TGT_UPDATE) -CRT_RPC_DEFINE(pool_add_in_v6, DAOS_ISEQ_POOL_TGT_UPDATE_V6, DAOS_OSEQ_POOL_TGT_UPDATE) CRT_RPC_DEFINE(pool_add_in, DAOS_ISEQ_POOL_TGT_UPDATE, DAOS_OSEQ_POOL_TGT_UPDATE) -CRT_RPC_DEFINE(pool_exclude_v6, DAOS_ISEQ_POOL_TGT_UPDATE_V6, DAOS_OSEQ_POOL_TGT_UPDATE) CRT_RPC_DEFINE(pool_exclude, DAOS_ISEQ_POOL_TGT_UPDATE, DAOS_OSEQ_POOL_TGT_UPDATE) -CRT_RPC_DEFINE(pool_drain_v6, DAOS_ISEQ_POOL_TGT_UPDATE_V6, DAOS_OSEQ_POOL_TGT_UPDATE) CRT_RPC_DEFINE(pool_drain, DAOS_ISEQ_POOL_TGT_UPDATE, DAOS_OSEQ_POOL_TGT_UPDATE) -CRT_RPC_DEFINE(pool_exclude_out_v6, DAOS_ISEQ_POOL_TGT_UPDATE_V6, DAOS_OSEQ_POOL_TGT_UPDATE) CRT_RPC_DEFINE(pool_exclude_out, DAOS_ISEQ_POOL_TGT_UPDATE, DAOS_OSEQ_POOL_TGT_UPDATE) -CRT_RPC_DEFINE(pool_evict, DAOS_ISEQ_POOL_EVICT_V6, DAOS_OSEQ_POOL_EVICT) -CRT_RPC_DEFINE(pool_svc_stop_v6, DAOS_ISEQ_POOL_SVC_STOP_V6, DAOS_OSEQ_POOL_SVC_STOP) +CRT_RPC_DEFINE(pool_evict, DAOS_ISEQ_POOL_EVICT, DAOS_OSEQ_POOL_EVICT) CRT_RPC_DEFINE(pool_svc_stop, DAOS_ISEQ_POOL_SVC_STOP, DAOS_OSEQ_POOL_SVC_STOP) CRT_RPC_DEFINE(pool_tgt_disconnect, DAOS_ISEQ_POOL_TGT_DISCONNECT, DAOS_OSEQ_POOL_TGT_DISCONNECT) -CRT_RPC_DEFINE(pool_tgt_query, DAOS_ISEQ_POOL_TGT_QUERY, - DAOS_OSEQ_POOL_TGT_QUERY) +CRT_RPC_DEFINE(pool_tgt_query_v6, DAOS_ISEQ_POOL_TGT_QUERY, DAOS_OSEQ_POOL_TGT_QUERY_V6) +CRT_RPC_DEFINE(pool_tgt_query, DAOS_ISEQ_POOL_TGT_QUERY, DAOS_OSEQ_POOL_TGT_QUERY) CRT_RPC_DEFINE(pool_tgt_dist_hdls, DAOS_ISEQ_POOL_TGT_DIST_HDLS, DAOS_OSEQ_POOL_TGT_DIST_HDLS) -/* NB: prop_get/set, and acl_update/delete, upgrade use V6 input. single version engine<->engine */ -CRT_RPC_DEFINE(pool_prop_get, DAOS_ISEQ_POOL_PROP_GET_V6, DAOS_OSEQ_POOL_PROP_GET) -CRT_RPC_DEFINE(pool_prop_set, DAOS_ISEQ_POOL_PROP_SET_V6, DAOS_OSEQ_POOL_PROP_SET) -CRT_RPC_DEFINE(pool_acl_update, DAOS_ISEQ_POOL_ACL_UPDATE_V6, DAOS_OSEQ_POOL_ACL_UPDATE) -CRT_RPC_DEFINE(pool_acl_delete, DAOS_ISEQ_POOL_ACL_DELETE_V6, DAOS_OSEQ_POOL_ACL_DELETE) +CRT_RPC_DEFINE(pool_prop_get, DAOS_ISEQ_POOL_PROP_GET, DAOS_OSEQ_POOL_PROP_GET) +CRT_RPC_DEFINE(pool_prop_set, DAOS_ISEQ_POOL_PROP_SET, DAOS_OSEQ_POOL_PROP_SET) +CRT_RPC_DEFINE(pool_acl_update, DAOS_ISEQ_POOL_ACL_UPDATE, DAOS_OSEQ_POOL_ACL_UPDATE) +CRT_RPC_DEFINE(pool_acl_delete, DAOS_ISEQ_POOL_ACL_DELETE, DAOS_OSEQ_POOL_ACL_DELETE) CRT_RPC_DEFINE(pool_ranks_get, DAOS_ISEQ_POOL_RANKS_GET, DAOS_OSEQ_POOL_RANKS_GET) -CRT_RPC_DEFINE(pool_upgrade, DAOS_ISEQ_POOL_UPGRADE_V6, DAOS_OSEQ_POOL_UPGRADE) -CRT_RPC_DEFINE(pool_list_cont_v6, DAOS_ISEQ_POOL_LIST_CONT_V6, DAOS_OSEQ_POOL_LIST_CONT) +CRT_RPC_DEFINE(pool_upgrade, DAOS_ISEQ_POOL_UPGRADE, DAOS_OSEQ_POOL_UPGRADE) CRT_RPC_DEFINE(pool_list_cont, DAOS_ISEQ_POOL_LIST_CONT, DAOS_OSEQ_POOL_LIST_CONT) static int @@ -214,9 +193,8 @@ crt_proc_daos_pool_cont_filter_t(crt_proc_t proc, crt_proc_op_t proc_op, return 0; } -CRT_RPC_DEFINE(pool_filter_cont_v6, DAOS_ISEQ_POOL_FILTER_CONT_V6, DAOS_OSEQ_POOL_FILTER_CONT) CRT_RPC_DEFINE(pool_filter_cont, DAOS_ISEQ_POOL_FILTER_CONT, DAOS_OSEQ_POOL_FILTER_CONT) -CRT_RPC_DEFINE(pool_query_info_v6, DAOS_ISEQ_POOL_QUERY_INFO_V6, DAOS_OSEQ_POOL_QUERY_INFO) +CRT_RPC_DEFINE(pool_query_info_v6, DAOS_ISEQ_POOL_QUERY_INFO, DAOS_OSEQ_POOL_QUERY_INFO_V6) CRT_RPC_DEFINE(pool_query_info, DAOS_ISEQ_POOL_QUERY_INFO, DAOS_OSEQ_POOL_QUERY_INFO) CRT_RPC_DEFINE(pool_tgt_query_map, DAOS_ISEQ_POOL_TGT_QUERY_MAP, DAOS_OSEQ_POOL_TGT_QUERY_MAP) CRT_RPC_DEFINE(pool_tgt_discard, DAOS_ISEQ_POOL_TGT_DISCARD, DAOS_OSEQ_POOL_TGT_DISCARD) @@ -234,29 +212,24 @@ CRT_RPC_DEFINE(pool_tgt_warmup, DAOS_ISEQ_POOL_TGT_WARMUP, DAOS_OSEQ_POOL_TGT_WA }, static struct crt_proto_rpc_format pool_proto_rpc_fmt_v6[] = {POOL_PROTO_CLI_RPC_LIST(6) - POOL_PROTO_SRV_RPC_LIST}; - -static struct crt_proto_rpc_format pool_proto_rpc_fmt_v5[] = { - POOL_PROTO_CLI_RPC_LIST(5) - POOL_PROTO_SRV_RPC_LIST -}; + POOL_PROTO_SRV_RPC_LIST(6)}; +static struct crt_proto_rpc_format pool_proto_rpc_fmt_v7[] = {POOL_PROTO_CLI_RPC_LIST(7) + POOL_PROTO_SRV_RPC_LIST(7)}; #undef X -struct crt_proto_format pool_proto_fmt_v5 = { - .cpf_name = "pool", - .cpf_ver = 5, - .cpf_count = ARRAY_SIZE(pool_proto_rpc_fmt_v5), - .cpf_prf = pool_proto_rpc_fmt_v5, - .cpf_base = DAOS_RPC_OPCODE(0, DAOS_POOL_MODULE, 0) -}; - struct crt_proto_format pool_proto_fmt_v6 = {.cpf_name = "pool", .cpf_ver = 6, .cpf_count = ARRAY_SIZE(pool_proto_rpc_fmt_v6), .cpf_prf = pool_proto_rpc_fmt_v6, .cpf_base = DAOS_RPC_OPCODE(0, DAOS_POOL_MODULE, 0)}; +struct crt_proto_format pool_proto_fmt_v7 = {.cpf_name = "pool", + .cpf_ver = 7, + .cpf_count = ARRAY_SIZE(pool_proto_rpc_fmt_v7), + .cpf_prf = pool_proto_rpc_fmt_v7, + .cpf_base = DAOS_RPC_OPCODE(0, DAOS_POOL_MODULE, 0)}; + uint64_t pool_query_bits(daos_pool_info_t *po_info, daos_prop_t *prop) { diff --git a/src/pool/rpc.h b/src/pool/rpc.h index cfddcc48931..b84a5a8a7c4 100644 --- a/src/pool/rpc.h +++ b/src/pool/rpc.h @@ -28,7 +28,7 @@ * These are for daos_rpc::dr_opc and DAOS_RPC_OPCODE(opc, ...) rather than * crt_req_create(..., opc, ...). See src/include/daos/rpc.h. */ -#define DAOS_POOL_VERSION 6 +#define DAOS_POOL_VERSION 7 /* LIST of internal RPCS in form of: * OPCODE, flags, FMT, handler, corpc_hdlr, */ @@ -37,53 +37,40 @@ #define POOL_PROTO_CLI_RPC_LIST(ver) \ X(POOL_CREATE, 0, &CQF_pool_create, ds_pool_create_handler, NULL) \ - X(POOL_CONNECT, 0, ver >= 6 ? &CQF_pool_connect_v6 : &CQF_pool_connect, \ - ver >= 6 ? ds_pool_connect_handler_v6 : ds_pool_connect_handler_v5, NULL) \ - X(POOL_DISCONNECT, 0, ver >= 6 ? &CQF_pool_disconnect_v6 : &CQF_pool_disconnect, \ - ver >= 6 ? ds_pool_disconnect_handler_v6 : ds_pool_disconnect_handler_v5, NULL) \ - X(POOL_QUERY, 0, ver >= 6 ? &CQF_pool_query_v6 : &CQF_pool_query, \ - ver >= 6 ? ds_pool_query_handler_v6 : ds_pool_query_handler_v5, NULL) \ - X(POOL_QUERY_INFO, 0, ver >= 6 ? &CQF_pool_query_info_v6 : &CQF_pool_query_info, \ - ver >= 6 ? ds_pool_query_info_handler_v6 : ds_pool_query_info_handler_v5, NULL) \ - X(POOL_EXCLUDE, 0, ver >= 6 ? &CQF_pool_exclude_v6 : &CQF_pool_exclude, \ - ver >= 6 ? ds_pool_update_handler_v6 : ds_pool_update_handler_v5, NULL) \ - X(POOL_DRAIN, 0, ver >= 6 ? &CQF_pool_drain_v6 : &CQF_pool_drain, \ - ver >= 6 ? ds_pool_update_handler_v6 : ds_pool_update_handler_v5, NULL) \ + X(POOL_CONNECT, 0, &CQF_pool_connect, ds_pool_connect_handler, NULL) \ + X(POOL_DISCONNECT, 0, &CQF_pool_disconnect, ds_pool_disconnect_handler, NULL) \ + X(POOL_QUERY, 0, ver >= 7 ? &CQF_pool_query : &CQF_pool_query_v6, \ + ver >= 7 ? ds_pool_query_handler : ds_pool_query_handler_v6, NULL) \ + X(POOL_QUERY_INFO, 0, ver >= 7 ? &CQF_pool_query_info : &CQF_pool_query_info_v6, \ + ver >= 7 ? ds_pool_query_info_handler : ds_pool_query_info_handler_v6, NULL) \ + X(POOL_EXCLUDE, 0, &CQF_pool_exclude, ds_pool_update_handler, NULL) \ + X(POOL_DRAIN, 0, &CQF_pool_drain, ds_pool_update_handler, NULL) \ X(POOL_EXTEND, 0, &CQF_pool_extend, ds_pool_extend_handler, NULL) \ X(POOL_EVICT, 0, &CQF_pool_evict, ds_pool_evict_handler, NULL) \ - X(POOL_REINT, 0, ver >= 6 ? &CQF_pool_add_v6 : &CQF_pool_add, \ - ver >= 6 ? ds_pool_update_handler_v6 : ds_pool_update_handler_v5, NULL) \ - X(POOL_ADD_IN, 0, ver >= 6 ? &CQF_pool_add_in_v6 : &CQF_pool_add_in, \ - ver >= 6 ? ds_pool_update_handler_v6 : ds_pool_update_handler_v5, NULL) \ - X(POOL_EXCLUDE_OUT, 0, ver >= 6 ? &CQF_pool_exclude_out_v6 : &CQF_pool_exclude_out, \ - ver >= 6 ? ds_pool_update_handler_v6 : ds_pool_update_handler_v5, NULL) \ - X(POOL_SVC_STOP, 0, ver >= 6 ? &CQF_pool_svc_stop_v6 : &CQF_pool_svc_stop, \ - ver >= 6 ? ds_pool_svc_stop_handler_v6 : ds_pool_svc_stop_handler_v5, NULL) \ - X(POOL_ATTR_LIST, 0, ver >= 6 ? &CQF_pool_attr_list_v6 : &CQF_pool_attr_list, \ - ver >= 6 ? ds_pool_attr_list_handler_v6 : ds_pool_attr_list_handler_v5, NULL) \ - X(POOL_ATTR_GET, 0, ver >= 6 ? &CQF_pool_attr_get_v6 : &CQF_pool_attr_get, \ - ver >= 6 ? ds_pool_attr_get_handler_v6 : ds_pool_attr_get_handler_v5, NULL) \ - X(POOL_ATTR_SET, 0, ver >= 6 ? &CQF_pool_attr_set_v6 : &CQF_pool_attr_set, \ - ver >= 6 ? ds_pool_attr_set_handler_v6 : ds_pool_attr_set_handler_v5, NULL) \ - X(POOL_ATTR_DEL, 0, ver >= 6 ? &CQF_pool_attr_del_v6 : &CQF_pool_attr_del, \ - ver >= 6 ? ds_pool_attr_del_handler_v6 : ds_pool_attr_del_handler_v5, NULL) \ + X(POOL_REINT, 0, &CQF_pool_add, ds_pool_update_handler, NULL) \ + X(POOL_ADD_IN, 0, &CQF_pool_add_in, ds_pool_update_handler, NULL) \ + X(POOL_EXCLUDE_OUT, 0, &CQF_pool_exclude_out, ds_pool_update_handler, NULL) \ + X(POOL_SVC_STOP, 0, &CQF_pool_svc_stop, ds_pool_svc_stop_handler, NULL) \ + X(POOL_ATTR_LIST, 0, &CQF_pool_attr_list, ds_pool_attr_list_handler, NULL) \ + X(POOL_ATTR_GET, 0, &CQF_pool_attr_get, ds_pool_attr_get_handler, NULL) \ + X(POOL_ATTR_SET, 0, &CQF_pool_attr_set, ds_pool_attr_set_handler, NULL) \ + X(POOL_ATTR_DEL, 0, &CQF_pool_attr_del, ds_pool_attr_del_handler, NULL) \ X(POOL_REPLICAS_ADD, 0, &CQF_pool_replicas_add, ds_pool_replicas_update_handler, NULL) \ X(POOL_REPLICAS_REMOVE, 0, &CQF_pool_replicas_remove, ds_pool_replicas_update_handler, \ NULL) \ - X(POOL_LIST_CONT, 0, ver >= 6 ? &CQF_pool_list_cont_v6 : &CQF_pool_list_cont, \ - ver >= 6 ? ds_pool_list_cont_handler_v6 : ds_pool_list_cont_handler_v5, NULL) \ + X(POOL_LIST_CONT, 0, &CQF_pool_list_cont, ds_pool_list_cont_handler, NULL) \ X(POOL_TGT_QUERY_MAP, 0, &CQF_pool_tgt_query_map, ds_pool_tgt_query_map_handler, NULL) \ - X(POOL_FILTER_CONT, 0, ver >= 6 ? &CQF_pool_filter_cont_v6 : &CQF_pool_filter_cont, \ - ver >= 6 ? ds_pool_filter_cont_handler_v6 : ds_pool_filter_cont_handler_v5, NULL) \ + X(POOL_FILTER_CONT, 0, &CQF_pool_filter_cont, ds_pool_filter_cont_handler, NULL) \ X(POOL_TGT_WARMUP, 0, &CQF_pool_tgt_warmup, ds_pool_tgt_warmup_handler, NULL) -#define POOL_PROTO_SRV_RPC_LIST \ +#define POOL_PROTO_SRV_RPC_LIST(ver) \ X(POOL_TGT_DISCONNECT, 0, &CQF_pool_tgt_disconnect, ds_pool_tgt_disconnect_handler, \ &ds_pool_tgt_disconnect_co_ops) \ - X(POOL_TGT_QUERY, 0, &CQF_pool_tgt_query, ds_pool_tgt_query_handler, \ - &ds_pool_tgt_query_co_ops) \ + X(POOL_TGT_QUERY, 0, ver >= 7 ? &CQF_pool_tgt_query : &CQF_pool_tgt_query_v6, \ + ver >= 7 ? ds_pool_tgt_query_handler : ds_pool_tgt_query_handler_v6, \ + ver >= 7 ? &ds_pool_tgt_query_co_ops : &ds_pool_tgt_query_co_ops_v6) \ X(POOL_PROP_GET, 0, &CQF_pool_prop_get, ds_pool_prop_get_handler, NULL) \ - X(POOL_ADD_TGT, 0, &CQF_pool_add_v6, ds_pool_update_handler_v6, NULL) \ + X(POOL_ADD_TGT, 0, &CQF_pool_add, ds_pool_update_handler, NULL) \ X(POOL_PROP_SET, 0, &CQF_pool_prop_set, ds_pool_prop_set_handler, NULL) \ X(POOL_ACL_UPDATE, 0, &CQF_pool_acl_update, ds_pool_acl_update_handler, NULL) \ X(POOL_ACL_DELETE, 0, &CQF_pool_acl_delete, ds_pool_acl_delete_handler, NULL) \ @@ -91,50 +78,46 @@ X(POOL_UPGRADE, 0, &CQF_pool_upgrade, ds_pool_upgrade_handler, NULL) \ X(POOL_TGT_DISCARD, 0, &CQF_pool_tgt_discard, ds_pool_tgt_discard_handler, NULL) -#define POOL_PROTO_RPC_LIST \ - POOL_PROTO_CLI_RPC_LIST(DAOS_POOL_VERSION) \ - POOL_PROTO_SRV_RPC_LIST +#define POOL_PROTO_RPC_LIST \ + POOL_PROTO_CLI_RPC_LIST(DAOS_POOL_VERSION) \ + POOL_PROTO_SRV_RPC_LIST(DAOS_POOL_VERSION) /* Define for RPC enum population below */ #define X(a, b, c, d, e) a, enum pool_operation { /* This list must stay consistent with POOL_PROTO_RPC_LIST. */ - POOL_PROTO_CLI_RPC_LIST(DAOS_POOL_VERSION) - POOL_PROTO_CLI_COUNT, + POOL_PROTO_CLI_RPC_LIST(DAOS_POOL_VERSION) POOL_PROTO_CLI_COUNT, POOL_PROTO_CLI_LAST = POOL_PROTO_CLI_COUNT - 1, - POOL_PROTO_SRV_RPC_LIST + POOL_PROTO_SRV_RPC_LIST(DAOS_POOL_VERSION) }; #undef X char *dc_pool_op_str(enum pool_operation op); -extern struct crt_proto_format pool_proto_fmt_v5; extern struct crt_proto_format pool_proto_fmt_v6; +extern struct crt_proto_format pool_proto_fmt_v7; extern int dc_pool_proto_version; /* clang-format off */ -#define DAOS_ISEQ_POOL_OP /* input fields */ \ +#define DAOS_ISEQ_POOL_OP /* input fields */ \ ((uuid_t) (pi_uuid) CRT_VAR) \ - ((uuid_t) (pi_hdl) CRT_VAR) - -#define DAOS_ISEQ_POOL_OP_V6 /* input fields */ \ - ((uuid_t)(pi_uuid)CRT_VAR)((uuid_t)(pi_hdl)CRT_VAR)((uuid_t)(pi_cli_id)CRT_VAR)( \ - (uint64_t)(pi_time)CRT_VAR) + ((uuid_t) (pi_hdl) CRT_VAR) \ + ((uuid_t) (pi_cli_id) CRT_VAR) \ + ((uint64_t) (pi_time) CRT_VAR) #define DAOS_OSEQ_POOL_OP /* output fields */ \ ((int32_t) (po_rc) CRT_VAR) \ ((uint32_t) (po_map_version) CRT_VAR) \ ((struct rsvc_hint) (po_hint) CRT_VAR) -CRT_RPC_DECLARE(pool_op_v6, DAOS_ISEQ_POOL_OP_V6, DAOS_OSEQ_POOL_OP) CRT_RPC_DECLARE(pool_op, DAOS_ISEQ_POOL_OP, DAOS_OSEQ_POOL_OP) /* If pri_op.pi_hdl is not null, call rdb_campaign. */ #define DAOS_ISEQ_POOL_CREATE /* input fields */ \ - ((struct pool_op_v6_in) (pri_op) CRT_VAR) \ + ((struct pool_op_in) (pri_op) CRT_VAR) \ ((d_rank_list_t) (pri_tgt_ranks) CRT_PTR) \ ((daos_prop_t) (pri_prop) CRT_PTR) \ ((uint32_t) (pri_ndomains) CRT_VAR) \ @@ -241,7 +224,7 @@ pool_create_in_set_data(crt_rpc_t *rpc, d_rank_list_t *pri_tgt_ranks, daos_prop_ /* only set on -DER_TRUNC */ \ ((uint32_t) (pco_map_buf_size) CRT_VAR) -#define DAOS_ISEQ_POOL_CONNECT /* input fields */ \ +#define DAOS_ISEQ_POOL_CONNECT /* input fields */ \ ((struct pool_op_in) (pci_op) CRT_VAR) \ ((d_iov_t) (pci_cred) CRT_VAR) \ ((uint64_t) (pci_flags) CRT_VAR) \ @@ -249,16 +232,7 @@ pool_create_in_set_data(crt_rpc_t *rpc, d_rank_list_t *pri_tgt_ranks, daos_prop_ ((crt_bulk_t) (pci_map_bulk) CRT_VAR) \ ((uint32_t) (pci_pool_version) CRT_VAR) -#define DAOS_ISEQ_POOL_CONNECT_V6 /* input fields */ \ - ((struct pool_op_v6_in) (pci_op) CRT_VAR) \ - ((d_iov_t) (pci_cred) CRT_VAR) \ - ((uint64_t) (pci_flags) CRT_VAR) \ - ((uint64_t) (pci_query_bits) CRT_VAR) \ - ((crt_bulk_t) (pci_map_bulk) CRT_VAR) \ - ((uint32_t) (pci_pool_version) CRT_VAR) - CRT_RPC_DECLARE(pool_connect, DAOS_ISEQ_POOL_CONNECT, DAOS_OSEQ_POOL_CONNECT) -CRT_RPC_DECLARE(pool_connect_v6, DAOS_ISEQ_POOL_CONNECT_V6, DAOS_OSEQ_POOL_CONNECT) /* clang-format on */ @@ -271,58 +245,40 @@ rpc_ver_atleast(crt_rpc_t *rpc, int min_ver) static inline void pool_connect_in_get_cred(crt_rpc_t *rpc, d_iov_t **pci_credp) { - void *in = crt_req_get(rpc); + struct pool_connect_in *in = crt_req_get(rpc); - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) - *pci_credp = &(((struct pool_connect_v6_in *)in)->pci_cred); - else - *pci_credp = &(((struct pool_connect_in *)in)->pci_cred); + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + *pci_credp = &in->pci_cred; } static inline void pool_connect_in_get_data(crt_rpc_t *rpc, uint64_t *pci_flagsp, uint64_t *pci_query_bitsp, crt_bulk_t *pci_map_bulkp, uint32_t *pci_pool_versionp) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - if (pci_flagsp) - *pci_flagsp = ((struct pool_connect_v6_in *)in)->pci_flags; - if (pci_query_bitsp) - *pci_query_bitsp = ((struct pool_connect_v6_in *)in)->pci_query_bits; - if (pci_map_bulkp) - *pci_map_bulkp = ((struct pool_connect_v6_in *)in)->pci_map_bulk; - if (pci_pool_versionp) - *pci_pool_versionp = ((struct pool_connect_v6_in *)in)->pci_pool_version; - } else { - if (pci_flagsp) - *pci_flagsp = ((struct pool_connect_in *)in)->pci_flags; - if (pci_query_bitsp) - *pci_query_bitsp = ((struct pool_connect_in *)in)->pci_query_bits; - if (pci_map_bulkp) - *pci_map_bulkp = ((struct pool_connect_in *)in)->pci_map_bulk; - if (pci_pool_versionp) - *pci_pool_versionp = ((struct pool_connect_in *)in)->pci_pool_version; - } + struct pool_connect_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + if (pci_flagsp) + *pci_flagsp = in->pci_flags; + if (pci_query_bitsp) + *pci_query_bitsp = in->pci_query_bits; + if (pci_map_bulkp) + *pci_map_bulkp = in->pci_map_bulk; + if (pci_pool_versionp) + *pci_pool_versionp = in->pci_pool_version; } static inline void pool_connect_in_set_data(crt_rpc_t *rpc, uint64_t pci_flags, uint64_t pci_query_bits, crt_bulk_t pci_map_bulk, uint32_t pci_pool_version) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - ((struct pool_connect_v6_in *)in)->pci_flags = pci_flags; - ((struct pool_connect_v6_in *)in)->pci_query_bits = pci_query_bits; - ((struct pool_connect_v6_in *)in)->pci_map_bulk = pci_map_bulk; - ((struct pool_connect_v6_in *)in)->pci_pool_version = pci_pool_version; - } else { - ((struct pool_connect_in *)in)->pci_flags = pci_flags; - ((struct pool_connect_in *)in)->pci_query_bits = pci_query_bits; - ((struct pool_connect_in *)in)->pci_map_bulk = pci_map_bulk; - ((struct pool_connect_in *)in)->pci_pool_version = pci_pool_version; - } + struct pool_connect_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + in->pci_flags = pci_flags; + in->pci_query_bits = pci_query_bits; + in->pci_map_bulk = pci_map_bulk; + in->pci_pool_version = pci_pool_version; } /* clang-format off */ @@ -330,24 +286,25 @@ pool_connect_in_set_data(crt_rpc_t *rpc, uint64_t pci_flags, uint64_t pci_query_ #define DAOS_ISEQ_POOL_DISCONNECT /* input fields */ \ ((struct pool_op_in) (pdi_op) CRT_VAR) -#define DAOS_ISEQ_POOL_DISCONNECT_V6 /* input fields */ \ - ((struct pool_op_v6_in) (pdi_op) CRT_VAR) - #define DAOS_OSEQ_POOL_DISCONNECT /* output fields */ \ ((struct pool_op_out) (pdo_op) CRT_VAR) CRT_RPC_DECLARE(pool_disconnect, DAOS_ISEQ_POOL_DISCONNECT, DAOS_OSEQ_POOL_DISCONNECT) -CRT_RPC_DECLARE(pool_disconnect_v6, DAOS_ISEQ_POOL_DISCONNECT_V6, DAOS_OSEQ_POOL_DISCONNECT) #define DAOS_ISEQ_POOL_QUERY /* input fields */ \ ((struct pool_op_in) (pqi_op) CRT_VAR) \ ((crt_bulk_t) (pqi_map_bulk) CRT_VAR) \ ((uint64_t) (pqi_query_bits) CRT_VAR) -#define DAOS_ISEQ_POOL_QUERY_V6 /* input fields */ \ - ((struct pool_op_v6_in) (pqi_op) CRT_VAR) \ - ((crt_bulk_t) (pqi_map_bulk) CRT_VAR) \ - ((uint64_t) (pqi_query_bits) CRT_VAR) +#define DAOS_OSEQ_POOL_QUERY_V6 /* output fields */ \ + ((struct pool_op_out) (pqo_op) CRT_VAR) \ + ((daos_prop_t) (pqo_prop) CRT_PTR) \ + ((struct daos_pool_space) (pqo_space) CRT_RAW) \ + ((struct daos_rebuild_status) (pqo_rebuild_st) CRT_RAW) \ + /* only set on -DER_TRUNC */ \ + ((uint32_t) (pqo_map_buf_size) CRT_VAR) \ + ((uint32_t) (pqo_pool_layout_ver) CRT_VAR) \ + ((uint32_t) (pqo_upgrade_layout_ver) CRT_VAR) #define DAOS_OSEQ_POOL_QUERY /* output fields */ \ ((struct pool_op_out) (pqo_op) CRT_VAR) \ @@ -357,39 +314,33 @@ CRT_RPC_DECLARE(pool_disconnect_v6, DAOS_ISEQ_POOL_DISCONNECT_V6, DAOS_OSEQ_POOL /* only set on -DER_TRUNC */ \ ((uint32_t) (pqo_map_buf_size) CRT_VAR) \ ((uint32_t) (pqo_pool_layout_ver) CRT_VAR) \ - ((uint32_t) (pqo_upgrade_layout_ver) CRT_VAR) + ((uint32_t) (pqo_upgrade_layout_ver) CRT_VAR) \ + ((uint32_t) (pqo_padding) CRT_VAR) \ + ((uint64_t) (pqo_mem_file_bytes) CRT_VAR) +CRT_RPC_DECLARE(pool_query_v6, DAOS_ISEQ_POOL_QUERY, DAOS_OSEQ_POOL_QUERY_V6) CRT_RPC_DECLARE(pool_query, DAOS_ISEQ_POOL_QUERY, DAOS_OSEQ_POOL_QUERY) -CRT_RPC_DECLARE(pool_query_v6, DAOS_ISEQ_POOL_QUERY_V6, DAOS_OSEQ_POOL_QUERY) /* clang-format on */ static inline void pool_query_in_get_data(crt_rpc_t *rpc, crt_bulk_t *pqi_map_bulkp, uint64_t *pqi_query_bitsp) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - *pqi_map_bulkp = ((struct pool_query_v6_in *)in)->pqi_map_bulk; - *pqi_query_bitsp = ((struct pool_query_v6_in *)in)->pqi_query_bits; - } else { - *pqi_map_bulkp = ((struct pool_query_in *)in)->pqi_map_bulk; - *pqi_query_bitsp = ((struct pool_query_in *)in)->pqi_query_bits; - } + struct pool_query_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + *pqi_map_bulkp = in->pqi_map_bulk; + *pqi_query_bitsp = in->pqi_query_bits; } static inline void pool_query_in_set_data(crt_rpc_t *rpc, crt_bulk_t pqi_map_bulk, uint64_t pqi_query_bits) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - ((struct pool_query_v6_in *)in)->pqi_map_bulk = pqi_map_bulk; - ((struct pool_query_v6_in *)in)->pqi_query_bits = pqi_query_bits; - } else { - ((struct pool_query_in *)in)->pqi_map_bulk = pqi_map_bulk; - ((struct pool_query_in *)in)->pqi_query_bits = pqi_query_bits; - } + struct pool_query_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + in->pqi_map_bulk = pqi_map_bulk; + in->pqi_query_bits = pqi_query_bits; } /* clang-format off */ @@ -399,49 +350,45 @@ pool_query_in_set_data(crt_rpc_t *rpc, crt_bulk_t pqi_map_bulk, uint64_t pqi_que ((d_rank_t) (pqii_rank) CRT_VAR) \ ((uint32_t) (pqii_tgt) CRT_VAR) -#define DAOS_ISEQ_POOL_QUERY_INFO_V6 /* input fields */ \ - ((struct pool_op_v6_in) (pqii_op) CRT_VAR) \ - ((d_rank_t) (pqii_rank) CRT_VAR) \ - ((uint32_t) (pqii_tgt) CRT_VAR) +#define DAOS_OSEQ_POOL_QUERY_INFO_V6 /* output fields */ \ + ((struct pool_op_out) (pqio_op) CRT_VAR) \ + ((d_rank_t) (pqio_rank) CRT_VAR) \ + ((uint32_t) (pqio_tgt) CRT_VAR) \ + ((struct daos_space) (pqio_space) CRT_RAW) \ + ((daos_target_state_t) (pqio_state) CRT_VAR) #define DAOS_OSEQ_POOL_QUERY_INFO /* output fields */ \ ((struct pool_op_out) (pqio_op) CRT_VAR) \ ((d_rank_t) (pqio_rank) CRT_VAR) \ ((uint32_t) (pqio_tgt) CRT_VAR) \ ((struct daos_space) (pqio_space) CRT_RAW) \ - ((daos_target_state_t) (pqio_state) CRT_VAR) + ((daos_target_state_t) (pqio_state) CRT_VAR) \ + ((uint32_t) (pqio_padding) CRT_VAR) \ + ((uint64_t) (pqio_mem_file_bytes) CRT_VAR) +CRT_RPC_DECLARE(pool_query_info_v6, DAOS_ISEQ_POOL_QUERY_INFO, DAOS_OSEQ_POOL_QUERY_INFO_V6) CRT_RPC_DECLARE(pool_query_info, DAOS_ISEQ_POOL_QUERY_INFO, DAOS_OSEQ_POOL_QUERY_INFO) -CRT_RPC_DECLARE(pool_query_info_v6, DAOS_ISEQ_POOL_QUERY_INFO_V6, DAOS_OSEQ_POOL_QUERY_INFO) /* clang-format on */ static inline void pool_query_info_in_get_data(crt_rpc_t *rpc, d_rank_t *pqii_rankp, uint32_t *pqii_tgtp) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - *pqii_rankp = ((struct pool_query_info_v6_in *)in)->pqii_rank; - *pqii_tgtp = ((struct pool_query_info_v6_in *)in)->pqii_tgt; - } else { - *pqii_rankp = ((struct pool_query_info_in *)in)->pqii_rank; - *pqii_tgtp = ((struct pool_query_info_in *)in)->pqii_tgt; - } + struct pool_query_info_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + *pqii_rankp = in->pqii_rank; + *pqii_tgtp = in->pqii_tgt; } static inline void pool_query_info_in_set_data(crt_rpc_t *rpc, d_rank_t pqii_rank, uint32_t pqii_tgt) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - ((struct pool_query_info_v6_in *)in)->pqii_rank = pqii_rank; - ((struct pool_query_info_v6_in *)in)->pqii_tgt = pqii_tgt; - } else { - ((struct pool_query_info_in *)in)->pqii_rank = pqii_rank; - ((struct pool_query_info_in *)in)->pqii_tgt = pqii_tgt; - } + struct pool_query_info_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + in->pqii_rank = pqii_rank; + in->pqii_tgt = pqii_tgt; } /* clang-format off */ @@ -450,16 +397,11 @@ pool_query_info_in_set_data(crt_rpc_t *rpc, d_rank_t pqii_rank, uint32_t pqii_tg ((struct pool_op_in) (pali_op) CRT_VAR) \ ((crt_bulk_t) (pali_bulk) CRT_VAR) -#define DAOS_ISEQ_POOL_ATTR_LIST_V6 /* input fields */ \ - ((struct pool_op_v6_in) (pali_op) CRT_VAR) \ - ((crt_bulk_t) (pali_bulk) CRT_VAR) - #define DAOS_OSEQ_POOL_ATTR_LIST /* output fields */ \ ((struct pool_op_out) (palo_op) CRT_VAR) \ ((uint64_t) (palo_size) CRT_VAR) CRT_RPC_DECLARE(pool_attr_list, DAOS_ISEQ_POOL_ATTR_LIST, DAOS_OSEQ_POOL_ATTR_LIST) -CRT_RPC_DECLARE(pool_attr_list_v6, DAOS_ISEQ_POOL_ATTR_LIST_V6, DAOS_OSEQ_POOL_ATTR_LIST) #define DAOS_ISEQ_POOL_TGT_WARMUP /* input fields */ \ ((crt_bulk_t) (tw_bulk) CRT_VAR) @@ -472,25 +414,19 @@ CRT_RPC_DECLARE(pool_tgt_warmup, DAOS_ISEQ_POOL_TGT_WARMUP, DAOS_OSEQ_POOL_TGT_W static inline void pool_attr_list_in_get_data(crt_rpc_t *rpc, crt_bulk_t *pali_bulkp) { - void *in = crt_req_get(rpc); + struct pool_attr_list_in *in = crt_req_get(rpc); - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - *pali_bulkp = ((struct pool_attr_list_v6_in *)in)->pali_bulk; - } else { - *pali_bulkp = ((struct pool_attr_list_in *)in)->pali_bulk; - } + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + *pali_bulkp = in->pali_bulk; } static inline void pool_attr_list_in_set_data(crt_rpc_t *rpc, crt_bulk_t pali_bulk) { - void *in = crt_req_get(rpc); + struct pool_attr_list_in *in = crt_req_get(rpc); - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - ((struct pool_attr_list_v6_in *)in)->pali_bulk = pali_bulk; - } else { - ((struct pool_attr_list_in *)in)->pali_bulk = pali_bulk; - } + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + in->pali_bulk = pali_bulk; } /* clang-format off */ @@ -501,14 +437,7 @@ pool_attr_list_in_set_data(crt_rpc_t *rpc, crt_bulk_t pali_bulk) ((uint64_t) (pagi_key_length) CRT_VAR) \ ((crt_bulk_t) (pagi_bulk) CRT_VAR) -#define DAOS_ISEQ_POOL_ATTR_GET_V6 /* input fields */ \ - ((struct pool_op_v6_in) (pagi_op) CRT_VAR) \ - ((uint64_t) (pagi_count) CRT_VAR) \ - ((uint64_t) (pagi_key_length) CRT_VAR) \ - ((crt_bulk_t) (pagi_bulk) CRT_VAR) - CRT_RPC_DECLARE(pool_attr_get, DAOS_ISEQ_POOL_ATTR_GET, DAOS_OSEQ_POOL_OP) -CRT_RPC_DECLARE(pool_attr_get_v6, DAOS_ISEQ_POOL_ATTR_GET_V6, DAOS_OSEQ_POOL_OP) /* clang-format on */ @@ -516,17 +445,12 @@ static inline void pool_attr_get_in_get_data(crt_rpc_t *rpc, uint64_t *pagi_countp, uint64_t *pagi_key_lengthp, crt_bulk_t *pagi_bulkp) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - *pagi_countp = ((struct pool_attr_get_v6_in *)in)->pagi_count; - *pagi_key_lengthp = ((struct pool_attr_get_v6_in *)in)->pagi_key_length; - *pagi_bulkp = ((struct pool_attr_get_v6_in *)in)->pagi_bulk; - } else { - *pagi_countp = ((struct pool_attr_get_in *)in)->pagi_count; - *pagi_key_lengthp = ((struct pool_attr_get_in *)in)->pagi_key_length; - *pagi_bulkp = ((struct pool_attr_get_in *)in)->pagi_bulk; - } + struct pool_attr_get_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + *pagi_countp = in->pagi_count; + *pagi_key_lengthp = in->pagi_key_length; + *pagi_bulkp = in->pagi_bulk; } static inline void @@ -534,17 +458,12 @@ pool_attr_get_in_set_data(crt_rpc_t *rpc, uint64_t pagi_count, uint64_t pagi_key crt_bulk_t pagi_bulk) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - ((struct pool_attr_get_v6_in *)in)->pagi_count = pagi_count; - ((struct pool_attr_get_v6_in *)in)->pagi_key_length = pagi_key_length; - ((struct pool_attr_get_v6_in *)in)->pagi_bulk = pagi_bulk; - } else { - ((struct pool_attr_get_in *)in)->pagi_count = pagi_count; - ((struct pool_attr_get_in *)in)->pagi_key_length = pagi_key_length; - ((struct pool_attr_get_in *)in)->pagi_bulk = pagi_bulk; - } + struct pool_attr_get_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + in->pagi_count = pagi_count; + in->pagi_key_length = pagi_key_length; + in->pagi_bulk = pagi_bulk; } /* clang-format off */ @@ -554,42 +473,28 @@ pool_attr_get_in_set_data(crt_rpc_t *rpc, uint64_t pagi_count, uint64_t pagi_key ((uint64_t) (pasi_count) CRT_VAR) \ ((crt_bulk_t) (pasi_bulk) CRT_VAR) -#define DAOS_ISEQ_POOL_ATTR_SET_V6 /* input fields */ \ - ((struct pool_op_v6_in) (pasi_op) CRT_VAR) \ - ((uint64_t) (pasi_count) CRT_VAR) \ - ((crt_bulk_t) (pasi_bulk) CRT_VAR) - CRT_RPC_DECLARE(pool_attr_set, DAOS_ISEQ_POOL_ATTR_SET, DAOS_OSEQ_POOL_OP) -CRT_RPC_DECLARE(pool_attr_set_v6, DAOS_ISEQ_POOL_ATTR_SET_V6, DAOS_OSEQ_POOL_OP) /* clang-format on */ static inline void pool_attr_set_in_get_data(crt_rpc_t *rpc, uint64_t *pasi_countp, crt_bulk_t *pasi_bulkp) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - *pasi_countp = ((struct pool_attr_set_v6_in *)in)->pasi_count; - *pasi_bulkp = ((struct pool_attr_set_v6_in *)in)->pasi_bulk; - } else { - *pasi_countp = ((struct pool_attr_set_in *)in)->pasi_count; - *pasi_bulkp = ((struct pool_attr_set_in *)in)->pasi_bulk; - } + struct pool_attr_set_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + *pasi_countp = in->pasi_count; + *pasi_bulkp = in->pasi_bulk; } static inline void pool_attr_set_in_set_data(crt_rpc_t *rpc, uint64_t pasi_count, crt_bulk_t pasi_bulk) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - ((struct pool_attr_set_v6_in *)in)->pasi_count = pasi_count; - ((struct pool_attr_set_v6_in *)in)->pasi_bulk = pasi_bulk; - } else { - ((struct pool_attr_set_in *)in)->pasi_count = pasi_count; - ((struct pool_attr_set_in *)in)->pasi_bulk = pasi_bulk; - } + struct pool_attr_set_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + in->pasi_count = pasi_count; + in->pasi_bulk = pasi_bulk; } /* clang-format off */ @@ -599,42 +504,28 @@ pool_attr_set_in_set_data(crt_rpc_t *rpc, uint64_t pasi_count, crt_bulk_t pasi_b ((uint64_t) (padi_count) CRT_VAR) \ ((crt_bulk_t) (padi_bulk) CRT_VAR) -#define DAOS_ISEQ_POOL_ATTR_DEL_V6 /* input fields */ \ - ((struct pool_op_v6_in) (padi_op) CRT_VAR) \ - ((uint64_t) (padi_count) CRT_VAR) \ - ((crt_bulk_t) (padi_bulk) CRT_VAR) - CRT_RPC_DECLARE(pool_attr_del, DAOS_ISEQ_POOL_ATTR_DEL, DAOS_OSEQ_POOL_OP) -CRT_RPC_DECLARE(pool_attr_del_v6, DAOS_ISEQ_POOL_ATTR_DEL_V6, DAOS_OSEQ_POOL_OP) /* clang-format on */ static inline void pool_attr_del_in_get_data(crt_rpc_t *rpc, uint64_t *padi_countp, crt_bulk_t *padi_bulkp) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - *padi_countp = ((struct pool_attr_del_v6_in *)in)->padi_count; - *padi_bulkp = ((struct pool_attr_del_v6_in *)in)->padi_bulk; - } else { - *padi_countp = ((struct pool_attr_del_in *)in)->padi_count; - *padi_bulkp = ((struct pool_attr_del_in *)in)->padi_bulk; - } + struct pool_attr_del_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + *padi_countp = in->padi_count; + *padi_bulkp = in->padi_bulk; } static inline void pool_attr_del_in_set_data(crt_rpc_t *rpc, uint64_t padi_count, crt_bulk_t padi_bulk) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - ((struct pool_attr_del_v6_in *)in)->padi_count = padi_count; - ((struct pool_attr_del_v6_in *)in)->padi_bulk = padi_bulk; - } else { - ((struct pool_attr_del_in *)in)->padi_count = padi_count; - ((struct pool_attr_del_in *)in)->padi_bulk = padi_bulk; - } + struct pool_attr_del_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + in->padi_count = padi_count; + in->padi_bulk = padi_bulk; } /* clang-format off */ @@ -656,16 +547,12 @@ CRT_RPC_DECLARE(pool_replicas_remove, DAOS_ISEQ_POOL_MEMBERSHIP, DAOS_OSEQ_POOL_ ((struct pool_op_in) (pti_op) CRT_VAR) \ ((struct pool_target_addr) (pti_addr_list) CRT_ARRAY) -#define DAOS_ISEQ_POOL_TGT_UPDATE_V6 /* input fields */ \ - ((struct pool_op_v6_in) (pti_op) CRT_VAR) \ - ((struct pool_target_addr) (pti_addr_list) CRT_ARRAY) - #define DAOS_OSEQ_POOL_TGT_UPDATE /* output fields */ \ ((struct pool_op_out) (pto_op) CRT_VAR) \ ((struct pool_target_addr) (pto_addr_list) CRT_ARRAY) -#define DAOS_ISEQ_POOL_EXTEND_V6 /* input fields */ \ - ((struct pool_op_v6_in) (pei_op) CRT_VAR) \ +#define DAOS_ISEQ_POOL_EXTEND /* input fields */ \ + ((struct pool_op_in) (pei_op) CRT_VAR) \ ((d_rank_list_t) (pei_tgt_ranks) CRT_PTR) \ ((uint32_t) (pei_ntgts) CRT_VAR) \ ((uint32_t) (pei_ndomains) CRT_VAR) \ @@ -675,54 +562,40 @@ CRT_RPC_DECLARE(pool_replicas_remove, DAOS_ISEQ_POOL_MEMBERSHIP, DAOS_OSEQ_POOL_ ((struct pool_op_out) (peo_op) CRT_VAR) CRT_RPC_DECLARE(pool_tgt_update, DAOS_ISEQ_POOL_TGT_UPDATE, DAOS_OSEQ_POOL_TGT_UPDATE) -CRT_RPC_DECLARE(pool_tgt_update_v6, DAOS_ISEQ_POOL_TGT_UPDATE_V6, DAOS_OSEQ_POOL_TGT_UPDATE) -CRT_RPC_DECLARE(pool_extend, DAOS_ISEQ_POOL_EXTEND_V6, DAOS_OSEQ_POOL_EXTEND) +CRT_RPC_DECLARE(pool_extend, DAOS_ISEQ_POOL_EXTEND, DAOS_OSEQ_POOL_EXTEND) CRT_RPC_DECLARE(pool_add, DAOS_ISEQ_POOL_TGT_UPDATE, DAOS_OSEQ_POOL_TGT_UPDATE) -CRT_RPC_DECLARE(pool_add_v6, DAOS_ISEQ_POOL_TGT_UPDATE_V6, DAOS_OSEQ_POOL_TGT_UPDATE) CRT_RPC_DECLARE(pool_add_in, DAOS_ISEQ_POOL_TGT_UPDATE, DAOS_OSEQ_POOL_TGT_UPDATE) -CRT_RPC_DECLARE(pool_add_in_v6, DAOS_ISEQ_POOL_TGT_UPDATE_V6, DAOS_OSEQ_POOL_TGT_UPDATE) CRT_RPC_DECLARE(pool_exclude, DAOS_ISEQ_POOL_TGT_UPDATE, DAOS_OSEQ_POOL_TGT_UPDATE) -CRT_RPC_DECLARE(pool_exclude_v6, DAOS_ISEQ_POOL_TGT_UPDATE_V6, DAOS_OSEQ_POOL_TGT_UPDATE) CRT_RPC_DECLARE(pool_drain, DAOS_ISEQ_POOL_TGT_UPDATE, DAOS_OSEQ_POOL_TGT_UPDATE) -CRT_RPC_DECLARE(pool_drain_v6, DAOS_ISEQ_POOL_TGT_UPDATE_V6, DAOS_OSEQ_POOL_TGT_UPDATE) CRT_RPC_DECLARE(pool_exclude_out, DAOS_ISEQ_POOL_TGT_UPDATE, DAOS_OSEQ_POOL_TGT_UPDATE) -CRT_RPC_DECLARE(pool_exclude_out_v6, DAOS_ISEQ_POOL_TGT_UPDATE_V6, DAOS_OSEQ_POOL_TGT_UPDATE) /* clang-format on */ static inline void pool_tgt_update_in_get_data(crt_rpc_t *rpc, struct pool_target_addr **pti_addr_listp, int *countp) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - *pti_addr_listp = ((struct pool_tgt_update_v6_in *)in)->pti_addr_list.ca_arrays; - *countp = (int)((struct pool_tgt_update_v6_in *)in)->pti_addr_list.ca_count; - } else { - *pti_addr_listp = ((struct pool_tgt_update_in *)in)->pti_addr_list.ca_arrays; - *countp = (int)((struct pool_tgt_update_in *)in)->pti_addr_list.ca_count; - } + struct pool_tgt_update_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + *pti_addr_listp = in->pti_addr_list.ca_arrays; + *countp = in->pti_addr_list.ca_count; } static inline void pool_tgt_update_in_set_data(crt_rpc_t *rpc, struct pool_target_addr *pti_addr_list, uint64_t count) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - ((struct pool_tgt_update_v6_in *)in)->pti_addr_list.ca_arrays = pti_addr_list; - ((struct pool_tgt_update_v6_in *)in)->pti_addr_list.ca_count = count; - } else { - ((struct pool_tgt_update_in *)in)->pti_addr_list.ca_arrays = pti_addr_list; - ((struct pool_tgt_update_in *)in)->pti_addr_list.ca_count = count; - } + struct pool_tgt_update_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + in->pti_addr_list.ca_arrays = pti_addr_list; + in->pti_addr_list.ca_count = count; } /* clang-format off */ -#define DAOS_ISEQ_POOL_EVICT_V6 /* input fields */ \ - ((struct pool_op_v6_in) (pvi_op) CRT_VAR) \ +#define DAOS_ISEQ_POOL_EVICT /* input fields */ \ + ((struct pool_op_in) (pvi_op) CRT_VAR) \ ((uint32_t) (pvi_pool_destroy) CRT_VAR) \ ((uint32_t) (pvi_pool_destroy_force) CRT_VAR) \ ((uuid_t) (pvi_hdls) CRT_ARRAY) \ @@ -732,19 +605,15 @@ pool_tgt_update_in_set_data(crt_rpc_t *rpc, struct pool_target_addr *pti_addr_li ((struct pool_op_out) (pvo_op) CRT_VAR) \ ((uint32_t) (pvo_n_hdls_evicted) CRT_VAR) -CRT_RPC_DECLARE(pool_evict, DAOS_ISEQ_POOL_EVICT_V6, DAOS_OSEQ_POOL_EVICT) +CRT_RPC_DECLARE(pool_evict, DAOS_ISEQ_POOL_EVICT, DAOS_OSEQ_POOL_EVICT) #define DAOS_ISEQ_POOL_SVC_STOP /* input fields */ \ ((struct pool_op_in) (psi_op) CRT_VAR) -#define DAOS_ISEQ_POOL_SVC_STOP_V6 /* input fields */ \ - ((struct pool_op_v6_in) (psi_op) CRT_VAR) - #define DAOS_OSEQ_POOL_SVC_STOP /* output fields */ \ ((struct pool_op_out) (pso_op) CRT_VAR) CRT_RPC_DECLARE(pool_svc_stop, DAOS_ISEQ_POOL_SVC_STOP, DAOS_OSEQ_POOL_SVC_STOP) -CRT_RPC_DECLARE(pool_svc_stop_v6, DAOS_ISEQ_POOL_SVC_STOP_V6, DAOS_OSEQ_POOL_SVC_STOP) #define DAOS_ISEQ_POOL_TGT_DISCONNECT /* input fields */ \ ((uuid_t) (tdi_uuid) CRT_VAR) \ @@ -758,10 +627,17 @@ CRT_RPC_DECLARE(pool_tgt_disconnect, DAOS_ISEQ_POOL_TGT_DISCONNECT, DAOS_OSEQ_PO #define DAOS_ISEQ_POOL_TGT_QUERY /* input fields */ \ ((struct pool_op_in) (tqi_op) CRT_VAR) -#define DAOS_OSEQ_POOL_TGT_QUERY /* output fields */ \ +#define DAOS_OSEQ_POOL_TGT_QUERY_V6 /* output fields */ \ ((struct daos_pool_space) (tqo_space) CRT_RAW) \ ((uint32_t) (tqo_rc) CRT_VAR) +#define DAOS_OSEQ_POOL_TGT_QUERY /* output fields */ \ + ((struct daos_pool_space) (tqo_space) CRT_RAW) \ + ((uint32_t) (tqo_rc) CRT_VAR) \ + ((uint32_t) (tqo_padding) CRT_VAR) \ + ((uint64_t) (tqo_mem_file_bytes) CRT_VAR) + +CRT_RPC_DECLARE(pool_tgt_query_v6, DAOS_ISEQ_POOL_TGT_QUERY, DAOS_OSEQ_POOL_TGT_QUERY_V6) CRT_RPC_DECLARE(pool_tgt_query, DAOS_ISEQ_POOL_TGT_QUERY, DAOS_OSEQ_POOL_TGT_QUERY) #define DAOS_ISEQ_POOL_TGT_DIST_HDLS /* input fields */ \ @@ -773,127 +649,127 @@ CRT_RPC_DECLARE(pool_tgt_query, DAOS_ISEQ_POOL_TGT_QUERY, DAOS_OSEQ_POOL_TGT_QUE CRT_RPC_DECLARE(pool_tgt_dist_hdls, DAOS_ISEQ_POOL_TGT_DIST_HDLS, DAOS_OSEQ_POOL_TGT_DIST_HDLS) -#define DAOS_ISEQ_POOL_PROP_GET_V6 /* input fields */ \ - ((struct pool_op_v6_in) (pgi_op) CRT_VAR) \ +#define DAOS_ISEQ_POOL_PROP_GET /* input fields */ \ + ((struct pool_op_in) (pgi_op) CRT_VAR) \ ((uint64_t) (pgi_query_bits) CRT_VAR) #define DAOS_OSEQ_POOL_PROP_GET /* output fields */ \ ((struct pool_op_out) (pgo_op) CRT_VAR) \ ((daos_prop_t) (pgo_prop) CRT_PTR) -CRT_RPC_DECLARE(pool_prop_get, DAOS_ISEQ_POOL_PROP_GET_V6, DAOS_OSEQ_POOL_PROP_GET) +CRT_RPC_DECLARE(pool_prop_get, DAOS_ISEQ_POOL_PROP_GET, DAOS_OSEQ_POOL_PROP_GET) /* clang-format on */ static inline void pool_prop_get_in_get_data(crt_rpc_t *rpc, uint64_t *pgi_query_bitsp) { - void *in = crt_req_get(rpc); + struct pool_prop_get_in *in = crt_req_get(rpc); /* engine<->engine RPC, assume same protocol version between them */ - *pgi_query_bitsp = ((struct pool_prop_get_in *)in)->pgi_query_bits; + *pgi_query_bitsp = in->pgi_query_bits; } static inline void pool_prop_get_in_set_data(crt_rpc_t *rpc, uint64_t pgi_query_bits) { - void *in = crt_req_get(rpc); + struct pool_prop_get_in *in = crt_req_get(rpc); - ((struct pool_prop_get_in *)in)->pgi_query_bits = pgi_query_bits; + in->pgi_query_bits = pgi_query_bits; } /* clang-format off */ -#define DAOS_ISEQ_POOL_PROP_SET_V6 /* input fields */ \ - ((struct pool_op_v6_in) (psi_op) CRT_VAR) \ +#define DAOS_ISEQ_POOL_PROP_SET /* input fields */ \ + ((struct pool_op_in) (psi_op) CRT_VAR) \ ((daos_prop_t) (psi_prop) CRT_PTR) #define DAOS_OSEQ_POOL_PROP_SET /* output fields */ \ ((struct pool_op_out) (pso_op) CRT_VAR) -CRT_RPC_DECLARE(pool_prop_set, DAOS_ISEQ_POOL_PROP_SET_V6, DAOS_OSEQ_POOL_PROP_SET) +CRT_RPC_DECLARE(pool_prop_set, DAOS_ISEQ_POOL_PROP_SET, DAOS_OSEQ_POOL_PROP_SET) /* clang-format on */ static inline void pool_prop_set_in_get_data(crt_rpc_t *rpc, daos_prop_t **psi_propp) { - void *in = crt_req_get(rpc); + struct pool_prop_set_in *in = crt_req_get(rpc); /* engine<->engine RPC, assume same protocol version between them */ - *psi_propp = ((struct pool_prop_set_in *)in)->psi_prop; + *psi_propp = in->psi_prop; } static inline void pool_prop_set_in_set_data(crt_rpc_t *rpc, daos_prop_t *psi_prop) { - void *in = crt_req_get(rpc); + struct pool_prop_set_in *in = crt_req_get(rpc); - ((struct pool_prop_set_in *)in)->psi_prop = psi_prop; + in->psi_prop = psi_prop; } /* clang-format off */ -#define DAOS_ISEQ_POOL_ACL_UPDATE_V6 /* input fields */ \ - ((struct pool_op_v6_in) (pui_op) CRT_VAR) \ +#define DAOS_ISEQ_POOL_ACL_UPDATE /* input fields */ \ + ((struct pool_op_in) (pui_op) CRT_VAR) \ ((struct daos_acl) (pui_acl) CRT_PTR) #define DAOS_OSEQ_POOL_ACL_UPDATE /* output fields */ \ ((struct pool_op_out) (puo_op) CRT_VAR) -CRT_RPC_DECLARE(pool_acl_update, DAOS_ISEQ_POOL_ACL_UPDATE_V6, DAOS_OSEQ_POOL_ACL_UPDATE) +CRT_RPC_DECLARE(pool_acl_update, DAOS_ISEQ_POOL_ACL_UPDATE, DAOS_OSEQ_POOL_ACL_UPDATE) /* clang-format on */ static inline void pool_acl_update_in_get_data(crt_rpc_t *rpc, struct daos_acl **pui_aclp) { - void *in = crt_req_get(rpc); + struct pool_acl_update_in *in = crt_req_get(rpc); /* engine<->engine RPC, assume same protocol version between them */ - *pui_aclp = ((struct pool_acl_update_in *)in)->pui_acl; + *pui_aclp = in->pui_acl; } static inline void pool_acl_update_in_set_data(crt_rpc_t *rpc, struct daos_acl *pui_acl) { - void *in = crt_req_get(rpc); + struct pool_acl_update_in *in = crt_req_get(rpc); - ((struct pool_acl_update_in *)in)->pui_acl = pui_acl; + in->pui_acl = pui_acl; } /* clang-format off */ -#define DAOS_ISEQ_POOL_ACL_DELETE_V6 /* input fields */ \ - ((struct pool_op_v6_in) (pdi_op) CRT_VAR) \ +#define DAOS_ISEQ_POOL_ACL_DELETE /* input fields */ \ + ((struct pool_op_in) (pdi_op) CRT_VAR) \ ((d_const_string_t) (pdi_principal) CRT_VAR) \ ((uint8_t) (pdi_type) CRT_VAR) #define DAOS_OSEQ_POOL_ACL_DELETE /* output fields */ \ ((struct pool_op_out) (pdo_op) CRT_VAR) -CRT_RPC_DECLARE(pool_acl_delete, DAOS_ISEQ_POOL_ACL_DELETE_V6, DAOS_OSEQ_POOL_ACL_DELETE) +CRT_RPC_DECLARE(pool_acl_delete, DAOS_ISEQ_POOL_ACL_DELETE, DAOS_OSEQ_POOL_ACL_DELETE) /* clang-format on */ static inline void pool_acl_delete_in_get_data(crt_rpc_t *rpc, d_const_string_t *pdi_principalp, uint8_t *pdi_typep) { - void *in = crt_req_get(rpc); + struct pool_acl_delete_in *in = crt_req_get(rpc); /* engine<->engine RPC, assume same protocol version between them */ - *pdi_principalp = ((struct pool_acl_delete_in *)in)->pdi_principal; - *pdi_typep = ((struct pool_acl_delete_in *)in)->pdi_type; + *pdi_principalp = in->pdi_principal; + *pdi_typep = in->pdi_type; } static inline void pool_acl_delete_in_set_data(crt_rpc_t *rpc, crt_opcode_t opc, d_const_string_t pdi_principal, uint8_t pdi_type) { - void *in = crt_req_get(rpc); + struct pool_acl_delete_in *in = crt_req_get(rpc); - ((struct pool_acl_delete_in *)in)->pdi_principal = pdi_principal; - ((struct pool_acl_delete_in *)in)->pdi_type = pdi_type; + in->pdi_principal = pdi_principal; + in->pdi_type = pdi_type; } /* clang-format off */ @@ -903,46 +779,32 @@ pool_acl_delete_in_set_data(crt_rpc_t *rpc, crt_opcode_t opc, d_const_string_t p ((crt_bulk_t) (plci_cont_bulk) CRT_VAR) \ ((uint64_t) (plci_ncont) CRT_VAR) -#define DAOS_ISEQ_POOL_LIST_CONT_V6 /* input fields */ \ - ((struct pool_op_v6_in) (plci_op) CRT_VAR) \ - ((crt_bulk_t) (plci_cont_bulk) CRT_VAR) \ - ((uint64_t) (plci_ncont) CRT_VAR) - #define DAOS_OSEQ_POOL_LIST_CONT /* output fields */ \ ((struct pool_op_out) (plco_op) CRT_VAR) \ ((uint64_t) (plco_ncont) CRT_VAR) CRT_RPC_DECLARE(pool_list_cont, DAOS_ISEQ_POOL_LIST_CONT, DAOS_OSEQ_POOL_LIST_CONT) -CRT_RPC_DECLARE(pool_list_cont_v6, DAOS_ISEQ_POOL_LIST_CONT_V6, DAOS_OSEQ_POOL_LIST_CONT) /* clang-format on */ static inline void pool_list_cont_in_get_data(crt_rpc_t *rpc, crt_bulk_t *plci_cont_bulkp, uint64_t *plci_ncontp) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - *plci_cont_bulkp = ((struct pool_list_cont_v6_in *)in)->plci_cont_bulk; - *plci_ncontp = ((struct pool_list_cont_v6_in *)in)->plci_ncont; - } else { - *plci_cont_bulkp = ((struct pool_list_cont_in *)in)->plci_cont_bulk; - *plci_ncontp = ((struct pool_list_cont_in *)in)->plci_ncont; - } + struct pool_list_cont_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + *plci_cont_bulkp = in->plci_cont_bulk; + *plci_ncontp = in->plci_ncont; } static inline void pool_list_cont_in_set_data(crt_rpc_t *rpc, crt_bulk_t plci_cont_bulk, uint64_t plci_ncont) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - ((struct pool_list_cont_v6_in *)in)->plci_cont_bulk = plci_cont_bulk; - ((struct pool_list_cont_v6_in *)in)->plci_ncont = plci_ncont; - } else { - ((struct pool_list_cont_in *)in)->plci_cont_bulk = plci_cont_bulk; - ((struct pool_list_cont_in *)in)->plci_ncont = plci_ncont; - } + struct pool_list_cont_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + in->plci_cont_bulk = plci_cont_bulk; + in->plci_ncont = plci_ncont; } /* clang-format off */ @@ -953,18 +815,11 @@ pool_list_cont_in_set_data(crt_rpc_t *rpc, crt_bulk_t plci_cont_bulk, uint64_t p ((uint64_t) (pfci_ncont) CRT_VAR) \ ((daos_pool_cont_filter_t) (pfci_filt) CRT_VAR) -#define DAOS_ISEQ_POOL_FILTER_CONT_V6 /* input fields */ \ - ((struct pool_op_v6_in) (pfci_op) CRT_VAR) \ - ((crt_bulk_t) (pfci_cont_bulk) CRT_VAR) \ - ((uint64_t) (pfci_ncont) CRT_VAR) \ - ((daos_pool_cont_filter_t) (pfci_filt) CRT_VAR) - #define DAOS_OSEQ_POOL_FILTER_CONT /* output fields */ \ ((struct pool_op_out) (pfco_op) CRT_VAR) \ ((uint64_t) (pfco_ncont) CRT_VAR) CRT_RPC_DECLARE(pool_filter_cont, DAOS_ISEQ_POOL_FILTER_CONT, DAOS_OSEQ_POOL_FILTER_CONT) -CRT_RPC_DECLARE(pool_filter_cont_v6, DAOS_ISEQ_POOL_FILTER_CONT_V6, DAOS_OSEQ_POOL_FILTER_CONT) /* clang-format on */ @@ -972,43 +827,27 @@ static inline void pool_filter_cont_in_get_data(crt_rpc_t *rpc, crt_bulk_t *pfci_cont_bulkp, uint64_t *pfci_ncontp, daos_pool_cont_filter_t **pfci_filtp) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - *pfci_cont_bulkp = ((struct pool_filter_cont_v6_in *)in)->pfci_cont_bulk; - *pfci_ncontp = ((struct pool_filter_cont_v6_in *)in)->pfci_ncont; - *pfci_filtp = &((struct pool_filter_cont_v6_in *)in)->pfci_filt; - } else { - *pfci_cont_bulkp = ((struct pool_filter_cont_in *)in)->pfci_cont_bulk; - *pfci_ncontp = ((struct pool_filter_cont_in *)in)->pfci_ncont; - *pfci_filtp = &((struct pool_filter_cont_in *)in)->pfci_filt; - } + struct pool_filter_cont_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + *pfci_cont_bulkp = in->pfci_cont_bulk; + *pfci_ncontp = in->pfci_ncont; + *pfci_filtp = &in->pfci_filt; } static inline void pool_filter_cont_in_set_data(crt_rpc_t *rpc, crt_bulk_t pfci_cont_bulk, uint64_t pfci_ncont, daos_pool_cont_filter_t *pfci_filt) { - void *in = crt_req_get(rpc); - - if (rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - ((struct pool_filter_cont_v6_in *)in)->pfci_cont_bulk = pfci_cont_bulk; - ((struct pool_filter_cont_v6_in *)in)->pfci_ncont = pfci_ncont; - if (pfci_filt) - ((struct pool_filter_cont_v6_in *)in)->pfci_filt = *pfci_filt; - else - memset(&((struct pool_filter_cont_v6_in *)in)->pfci_filt, 0, - sizeof(daos_pool_cont_filter_t)); - - } else { - ((struct pool_filter_cont_in *)in)->pfci_cont_bulk = pfci_cont_bulk; - ((struct pool_filter_cont_in *)in)->pfci_ncont = pfci_ncont; - if (pfci_filt) - ((struct pool_filter_cont_in *)in)->pfci_filt = *pfci_filt; - else - memset(&((struct pool_filter_cont_in *)in)->pfci_filt, 0, - sizeof(daos_pool_cont_filter_t)); - } + struct pool_filter_cont_in *in = crt_req_get(rpc); + + D_ASSERT(rpc_ver_atleast(rpc, POOL_PROTO_VER_WITH_SVC_OP_KEY)); + in->pfci_cont_bulk = pfci_cont_bulk; + in->pfci_ncont = pfci_ncont; + if (pfci_filt) + in->pfci_filt = *pfci_filt; + else + memset(&in->pfci_filt, 0, sizeof(daos_pool_cont_filter_t)); } /* clang-format off */ @@ -1024,13 +863,13 @@ pool_filter_cont_in_set_data(crt_rpc_t *rpc, crt_bulk_t pfci_cont_bulk, uint64_t CRT_RPC_DECLARE(pool_ranks_get, DAOS_ISEQ_POOL_RANKS_GET, DAOS_OSEQ_POOL_RANKS_GET) -#define DAOS_ISEQ_POOL_UPGRADE_V6 /* input fields */ \ - ((struct pool_op_v6_in) (poi_op) CRT_VAR) +#define DAOS_ISEQ_POOL_UPGRADE /* input fields */ \ + ((struct pool_op_in) (poi_op) CRT_VAR) #define DAOS_OSEQ_POOL_UPGRADE /* output fields */ \ ((struct pool_op_out) (poo_op) CRT_VAR) -CRT_RPC_DECLARE(pool_upgrade, DAOS_ISEQ_POOL_UPGRADE_V6, DAOS_OSEQ_POOL_UPGRADE) +CRT_RPC_DECLARE(pool_upgrade, DAOS_ISEQ_POOL_UPGRADE, DAOS_OSEQ_POOL_UPGRADE) #define DAOS_ISEQ_POOL_TGT_QUERY_MAP /* input fields */ \ ((struct pool_op_in) (tmi_op) CRT_VAR) \ @@ -1087,12 +926,10 @@ pool_req_create(crt_context_t crt_ctx, crt_endpoint_t *tgt_ep, crt_opcode_t opc, if (req_timep && (*req_timep == 0)) *req_timep = d_hlc_get(); - /* Temporary req_timep check: some opcodes aren't (yet) at v8 and don't have the op key */ - if (req_timep && (proto_ver >= POOL_PROTO_VER_WITH_SVC_OP_KEY)) { - struct pool_op_v6_in *in6 = crt_req_get(*req); - - uuid_copy(in6->pi_cli_id, cli_id); - in6->pi_time = *req_timep; + D_ASSERT(proto_ver >= POOL_PROTO_VER_WITH_SVC_OP_KEY); + if (req_timep) { + uuid_copy(in->pi_cli_id, cli_id); + in->pi_time = *req_timep; } return 0; } diff --git a/src/pool/srv.c b/src/pool/srv.c index 9c9611c04ef..3ecd9410ce8 100644 --- a/src/pool/srv.c +++ b/src/pool/srv.c @@ -133,6 +133,11 @@ static struct crt_corpc_ops ds_pool_tgt_disconnect_co_ops = { .co_pre_forward = NULL, }; +static struct crt_corpc_ops ds_pool_tgt_query_co_ops_v6 = { + .co_aggregate = ds_pool_tgt_query_aggregator_v6, + .co_pre_forward = NULL, +}; + static struct crt_corpc_ops ds_pool_tgt_query_co_ops = { .co_aggregate = ds_pool_tgt_query_aggregator, .co_pre_forward = NULL, @@ -148,11 +153,11 @@ static struct crt_corpc_ops ds_pool_tgt_query_co_ops = { .dr_corpc_ops = e, \ }, -static struct daos_rpc_handler pool_handlers_v5[] = {POOL_PROTO_CLI_RPC_LIST(5) - POOL_PROTO_SRV_RPC_LIST}; - static struct daos_rpc_handler pool_handlers_v6[] = {POOL_PROTO_CLI_RPC_LIST(6) - POOL_PROTO_SRV_RPC_LIST}; + POOL_PROTO_SRV_RPC_LIST(6)}; + +static struct daos_rpc_handler pool_handlers_v7[] = {POOL_PROTO_CLI_RPC_LIST(7) + POOL_PROTO_SRV_RPC_LIST(7)}; #undef X @@ -219,9 +224,9 @@ struct dss_module pool_module = { .sm_fini = fini, .sm_setup = setup, .sm_cleanup = cleanup, - .sm_proto_fmt = {&pool_proto_fmt_v5, &pool_proto_fmt_v6}, + .sm_proto_fmt = {&pool_proto_fmt_v6, &pool_proto_fmt_v7}, .sm_cli_count = {POOL_PROTO_CLI_COUNT, POOL_PROTO_CLI_COUNT}, - .sm_handlers = {pool_handlers_v5, pool_handlers_v6}, + .sm_handlers = {pool_handlers_v6, pool_handlers_v7}, .sm_key = &pool_module_key, .sm_metrics = &pool_metrics, }; diff --git a/src/pool/srv_cli.c b/src/pool/srv_cli.c index bbc228dd2b1..11ce8b80682 100644 --- a/src/pool/srv_cli.c +++ b/src/pool/srv_cli.c @@ -343,6 +343,7 @@ struct pool_query_arg { uint32_t *pqa_upgrade_layout_ver; crt_bulk_t pqa_bulk; struct pool_buf *pqa_map_buf; + uint64_t *pqa_mem_file_bytes; uint32_t pqa_map_size; }; @@ -541,6 +542,8 @@ pool_query_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) *arg->pqa_layout_ver = out->pqo_pool_layout_ver; if (arg->pqa_upgrade_layout_ver) *arg->pqa_upgrade_layout_ver = out->pqo_upgrade_layout_ver; + if (arg->pqa_mem_file_bytes) + *arg->pqa_mem_file_bytes = out->pqo_mem_file_bytes; if (rc != 0) D_ERROR(DF_UUID": failed to process pool query results, "DF_RC"\n", DP_UUID(pool_uuid), DP_RC(rc)); @@ -589,7 +592,8 @@ int dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, d_rank_list_t **enabled_ranks, d_rank_list_t **disabled_ranks, d_rank_list_t **dead_ranks, daos_pool_info_t *pool_info, - uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver) + uint32_t *pool_layout_ver, uint32_t *upgrade_layout_ver, + uint64_t *mem_file_bytes) { struct pool_query_arg arg = { .pqa_enabled_ranks = enabled_ranks, @@ -598,6 +602,7 @@ dsc_pool_svc_query(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, .pqa_info = pool_info, .pqa_layout_ver = pool_layout_ver, .pqa_upgrade_layout_ver = upgrade_layout_ver, + .pqa_mem_file_bytes = mem_file_bytes, .pqa_map_size = 127 /* 4 KB */ }; @@ -608,6 +613,7 @@ struct pool_query_target_arg { d_rank_t pqta_rank; uint32_t pqta_tgt_idx; daos_target_info_t *pqta_info; + uint64_t *pqta_mem_file_bytes; }; static int @@ -642,6 +648,8 @@ pool_query_target_consume(uuid_t pool_uuid, crt_rpc_t *rpc, void *varg) arg->pqta_info->ta_space.s_total[i] = out->pqio_space.s_total[i]; arg->pqta_info->ta_space.s_free[i] = out->pqio_space.s_free[i]; } + if (arg->pqta_mem_file_bytes) + *arg->pqta_mem_file_bytes = out->pqio_mem_file_bytes; return 0; } @@ -669,12 +677,14 @@ static struct dsc_pool_svc_call_cbs pool_query_target_cbs = { */ int dsc_pool_svc_query_target(uuid_t pool_uuid, d_rank_list_t *ps_ranks, uint64_t deadline, - d_rank_t rank, uint32_t tgt_idx, daos_target_info_t *ti) + d_rank_t rank, uint32_t tgt_idx, daos_target_info_t *ti, + uint64_t *mem_file_bytes) { struct pool_query_target_arg arg = { .pqta_rank = rank, .pqta_tgt_idx = tgt_idx, - .pqta_info = ti + .pqta_info = ti, + .pqta_mem_file_bytes = mem_file_bytes }; if (ti == NULL) diff --git a/src/pool/srv_internal.h b/src/pool/srv_internal.h index 612f7760fd1..628a20ef947 100644 --- a/src/pool/srv_internal.h +++ b/src/pool/srv_internal.h @@ -163,57 +163,39 @@ int ds_pool_hdl_is_from_srv(struct ds_pool *pool, uuid_t hdl); int ds_pool_svc_upgrade_vos_pool(struct ds_pool *pool); void ds_pool_create_handler(crt_rpc_t *rpc); void - ds_pool_connect_handler_v6(crt_rpc_t *rpc); -void ds_pool_connect_handler_v5(crt_rpc_t *rpc); +ds_pool_connect_handler(crt_rpc_t *rpc); void -ds_pool_disconnect_handler_v6(crt_rpc_t *rpc); +ds_pool_disconnect_handler(crt_rpc_t *rpc); void -ds_pool_disconnect_handler_v5(crt_rpc_t *rpc); +ds_pool_query_handler_v6(crt_rpc_t *rpc); void - ds_pool_query_handler_v6(crt_rpc_t *rpc); -void ds_pool_query_handler_v5(crt_rpc_t *rpc); + ds_pool_query_handler(crt_rpc_t *rpc); void ds_pool_prop_get_handler(crt_rpc_t *rpc); void ds_pool_prop_set_handler(crt_rpc_t *rpc); void ds_pool_acl_update_handler(crt_rpc_t *rpc); void ds_pool_acl_delete_handler(crt_rpc_t *rpc); void -ds_pool_update_handler_v6(crt_rpc_t *rpc); -void - ds_pool_update_handler_v5(crt_rpc_t *rpc); + ds_pool_update_handler(crt_rpc_t *rpc); void ds_pool_extend_handler(crt_rpc_t *rpc); void ds_pool_evict_handler(crt_rpc_t *rpc); void -ds_pool_svc_stop_handler_v6(crt_rpc_t *rpc); -void -ds_pool_svc_stop_handler_v5(crt_rpc_t *rpc); -void -ds_pool_attr_list_handler_v6(crt_rpc_t *rpc); -void -ds_pool_attr_list_handler_v5(crt_rpc_t *rpc); -void -ds_pool_attr_get_handler_v6(crt_rpc_t *rpc); +ds_pool_svc_stop_handler(crt_rpc_t *rpc); void -ds_pool_attr_get_handler_v5(crt_rpc_t *rpc); +ds_pool_attr_list_handler(crt_rpc_t *rpc); void -ds_pool_attr_set_handler_v6(crt_rpc_t *rpc); +ds_pool_attr_get_handler(crt_rpc_t *rpc); void -ds_pool_attr_set_handler_v5(crt_rpc_t *rpc); +ds_pool_attr_set_handler(crt_rpc_t *rpc); void -ds_pool_attr_del_handler_v6(crt_rpc_t *rpc); +ds_pool_attr_del_handler(crt_rpc_t *rpc); void -ds_pool_attr_del_handler_v5(crt_rpc_t *rpc); +ds_pool_list_cont_handler(crt_rpc_t *rpc); void -ds_pool_list_cont_handler_v6(crt_rpc_t *rpc); +ds_pool_filter_cont_handler(crt_rpc_t *rpc); void -ds_pool_list_cont_handler_v5(crt_rpc_t *rpc); + ds_pool_query_info_handler_v6(crt_rpc_t *rpc); void -ds_pool_filter_cont_handler_v6(crt_rpc_t *rpc); -void -ds_pool_filter_cont_handler_v5(crt_rpc_t *rpc); -void -ds_pool_query_info_handler_v6(crt_rpc_t *rpc); -void - ds_pool_query_info_handler_v5(crt_rpc_t *rpc); + ds_pool_query_info_handler(crt_rpc_t *rpc); void ds_pool_ranks_get_handler(crt_rpc_t *rpc); void ds_pool_upgrade_handler(crt_rpc_t *rpc); @@ -228,9 +210,13 @@ void ds_pool_hdl_hash_fini(void); void ds_pool_tgt_disconnect_handler(crt_rpc_t *rpc); int ds_pool_tgt_disconnect_aggregator(crt_rpc_t *source, crt_rpc_t *result, void *priv); +void + ds_pool_tgt_query_handler_v6(crt_rpc_t *rpc); void ds_pool_tgt_query_handler(crt_rpc_t *rpc); -int ds_pool_tgt_query_aggregator(crt_rpc_t *source, crt_rpc_t *result, - void *priv); +int +ds_pool_tgt_query_aggregator_v6(crt_rpc_t *source, crt_rpc_t *result, void *priv); +int + ds_pool_tgt_query_aggregator(crt_rpc_t *source, crt_rpc_t *result, void *priv); void ds_pool_replicas_update_handler(crt_rpc_t *rpc); int ds_pool_tgt_prop_update(struct ds_pool *pool, struct pool_iv_prop *iv_prop); int ds_pool_tgt_connect(struct ds_pool *pool, struct pool_iv_conn *pic); diff --git a/src/pool/srv_pool.c b/src/pool/srv_pool.c index d66ae12c328..06a5f29e866 100644 --- a/src/pool/srv_pool.c +++ b/src/pool/srv_pool.c @@ -239,8 +239,9 @@ D_LIST_HEAD(pool_svc_failed_list); static bool pool_disable_exclude; static int pool_prop_read(struct rdb_tx *tx, const struct pool_svc *svc, uint64_t bits, daos_prop_t **prop_out); -static int pool_space_query_bcast(crt_context_t ctx, struct pool_svc *svc, - uuid_t pool_hdl, struct daos_pool_space *ps); +static int + pool_space_query_bcast(crt_context_t ctx, struct pool_svc *svc, uuid_t pool_hdl, + struct daos_pool_space *ps, uint64_t *mem_file_bytes); static int ds_pool_upgrade_if_needed(uuid_t pool_uuid, struct rsvc_hint *po_hint, struct pool_svc *svc, crt_rpc_t *rpc); static int @@ -3495,19 +3496,16 @@ static int pool_op_lookup(struct rdb_tx *tx, struct pool_svc *svc, crt_rpc_t *rpc, int pool_proto_ver, bool *is_dup, struct ds_pool_svc_op_val *valp) { - struct pool_op_v6_in *in6 = crt_req_get(rpc); + struct pool_op_in *in = crt_req_get(rpc); crt_opcode_t opc = opc_get(rpc->cr_opc); int rc = 0; - /* If client didn't provide a key (old protocol), skip */ - if (pool_proto_ver < POOL_PROTO_VER_WITH_SVC_OP_KEY) - goto out; - + D_ASSERT(pool_proto_ver >= POOL_PROTO_VER_WITH_SVC_OP_KEY); /* If the operation is not a write, skip (read-only ops not tracked for duplicates) */ if (!pool_op_is_write(opc)) goto out; - rc = ds_pool_svc_ops_lookup(tx, svc, svc->ps_uuid, &in6->pi_cli_id, in6->pi_time, is_dup, + rc = ds_pool_svc_ops_lookup(tx, svc, svc->ps_uuid, &in->pi_cli_id, in->pi_time, is_dup, valp); out: @@ -3602,23 +3600,20 @@ static int pool_op_save(struct rdb_tx *tx, struct pool_svc *svc, crt_rpc_t *rpc, int pool_proto_ver, bool dup_op, int rc_in, struct ds_pool_svc_op_val *op_valp) { - struct pool_op_v6_in *in6 = crt_req_get(rpc); + struct pool_op_in *in = crt_req_get(rpc); crt_opcode_t opc = opc_get(rpc->cr_opc); int rc = 0; if (!dup_op) op_valp->ov_rc = rc_in; - /* If client didn't provide a key (old protocol), skip */ - if (pool_proto_ver < POOL_PROTO_VER_WITH_SVC_OP_KEY) - goto out; - + D_ASSERT(pool_proto_ver >= POOL_PROTO_VER_WITH_SVC_OP_KEY); /* If the operation is not a write, skip (read-only ops not tracked for duplicates) */ if (!pool_op_is_write(opc)) goto out; - rc = ds_pool_svc_ops_save(tx, svc, svc->ps_uuid, &in6->pi_cli_id, in6->pi_time, dup_op, - rc_in, op_valp); + rc = ds_pool_svc_ops_save(tx, svc, svc->ps_uuid, &in->pi_cli_id, in->pi_time, dup_op, rc_in, + op_valp); out: return rc; @@ -3827,7 +3822,7 @@ bulk_cb(const struct crt_bulk_cb_info *cb_info) #define NUM_POOL_VERSIONS 2 static void -ds_pool_connect_handler(crt_rpc_t *rpc, int handler_version) +pool_connect_handler(crt_rpc_t *rpc, int handler_version) { struct pool_connect_in *in = crt_req_get(rpc); struct pool_connect_out *out = crt_reply_get(rpc); @@ -3873,8 +3868,7 @@ ds_pool_connect_handler(crt_rpc_t *rpc, int handler_version) fi_pass_nl_noreply = DAOS_FAIL_CHECK(DAOS_MD_OP_PASS_NOREPLY_NEWLDR); fi_fail_nl_noreply = DAOS_FAIL_CHECK(DAOS_MD_OP_FAIL_NOREPLY_NEWLDR); - rc = pool_svc_lookup_leader(in->pci_op.pi_uuid, &svc, - &out->pco_op.po_hint); + rc = pool_svc_lookup_leader(in->pci_op.pi_uuid, &svc, &out->pco_op.po_hint); if (rc != 0) D_GOTO(out, rc); @@ -4162,7 +4156,8 @@ ds_pool_connect_handler(crt_rpc_t *rpc, int handler_version) } if ((rc == 0) && (query_bits & DAOS_PO_QUERY_SPACE)) - rc = pool_space_query_bcast(rpc->cr_ctx, svc, in->pci_op.pi_hdl, &out->pco_space); + rc = pool_space_query_bcast(rpc->cr_ctx, svc, in->pci_op.pi_hdl, &out->pco_space, + NULL); out_lock: ABT_rwlock_unlock(svc->ps_lock); @@ -4213,15 +4208,9 @@ ds_pool_connect_handler(crt_rpc_t *rpc, int handler_version) } void -ds_pool_connect_handler_v6(crt_rpc_t *rpc) -{ - ds_pool_connect_handler(rpc, 6); -} - -void -ds_pool_connect_handler_v5(crt_rpc_t *rpc) +ds_pool_connect_handler(crt_rpc_t *rpc) { - ds_pool_connect_handler(rpc, 5); + pool_connect_handler(rpc, DAOS_POOL_VERSION); } static int @@ -4327,7 +4316,7 @@ pool_disconnect_hdls(struct rdb_tx *tx, struct pool_svc *svc, uuid_t *hdl_uuids, } static void -ds_pool_disconnect_handler(crt_rpc_t *rpc, int handler_version) +pool_disconnect_handler(crt_rpc_t *rpc, int handler_version) { struct pool_disconnect_in *pdi = crt_req_get(rpc); struct pool_disconnect_out *pdo = crt_reply_get(rpc); @@ -4343,13 +4332,10 @@ ds_pool_disconnect_handler(crt_rpc_t *rpc, int handler_version) D_DEBUG(DB_MD, DF_UUID ": processing rpc: %p hdl=" DF_UUID "\n", DP_UUID(pdi->pdi_op.pi_uuid), rpc, DP_UUID(pdi->pdi_op.pi_hdl)); - if (handler_version >= POOL_PROTO_VER_WITH_SVC_OP_KEY) { - struct pool_disconnect_v6_in *in6 = crt_req_get(rpc); - D_DEBUG(DB_MD, DF_UUID ": client= " DF_UUID ", time=" DF_X64 "\n", - DP_UUID(pdi->pdi_op.pi_uuid), DP_UUID(in6->pdi_op.pi_cli_id), - in6->pdi_op.pi_time); - } + D_ASSERT(handler_version >= POOL_PROTO_VER_WITH_SVC_OP_KEY); + D_DEBUG(DB_MD, DF_UUID ": client= " DF_UUID ", time=" DF_X64 "\n", + DP_UUID(pdi->pdi_op.pi_uuid), DP_UUID(pdi->pdi_op.pi_cli_id), pdi->pdi_op.pi_time); rc = pool_svc_lookup_leader(pdi->pdi_op.pi_uuid, &svc, &pdo->pdo_op.po_hint); @@ -4429,20 +4415,14 @@ ds_pool_disconnect_handler(crt_rpc_t *rpc, int handler_version) } void -ds_pool_disconnect_handler_v6(crt_rpc_t *rpc) -{ - ds_pool_disconnect_handler(rpc, 6); -} - -void -ds_pool_disconnect_handler_v5(crt_rpc_t *rpc) +ds_pool_disconnect_handler(crt_rpc_t *rpc) { - ds_pool_disconnect_handler(rpc, 5); + pool_disconnect_handler(rpc, DAOS_POOL_VERSION); } static int pool_space_query_bcast(crt_context_t ctx, struct pool_svc *svc, uuid_t pool_hdl, - struct daos_pool_space *ps) + struct daos_pool_space *ps, uint64_t *mem_file_bytes) { struct pool_tgt_query_in *in; struct pool_tgt_query_out *out; @@ -4469,15 +4449,16 @@ pool_space_query_bcast(crt_context_t ctx, struct pool_svc *svc, uuid_t pool_hdl, out = crt_reply_get(rpc); rc = out->tqo_rc; - if (rc != 0) { + D_ASSERT(ps != NULL); + if (rc == 0) { + *ps = out->tqo_space; + if (mem_file_bytes != NULL) + *mem_file_bytes = out->tqo_mem_file_bytes; + } else { D_ERROR(DF_UUID ": failed to query from targets: " DF_RC "\n", DP_UUID(svc->ps_uuid), DP_RC(rc)); rc = -DER_IO; - } else { - D_ASSERT(ps != NULL); - *ps = out->tqo_space; } - out_rpc: crt_req_decref(rpc); out: @@ -4678,7 +4659,7 @@ static bool is_pool_from_srv(uuid_t pool_uuid, uuid_t poh_uuid); * Requires a pool handle (except for rebuild). */ static void -ds_pool_list_cont_handler(crt_rpc_t *rpc, int handler_version) +pool_list_cont_handler(crt_rpc_t *rpc, int handler_version) { struct pool_list_cont_in *in = crt_req_get(rpc); struct pool_list_cont_out *out = crt_reply_get(rpc); @@ -4770,15 +4751,9 @@ ds_pool_list_cont_handler(crt_rpc_t *rpc, int handler_version) } void -ds_pool_list_cont_handler_v6(crt_rpc_t *rpc) -{ - ds_pool_list_cont_handler(rpc, 6); -} - -void -ds_pool_list_cont_handler_v5(crt_rpc_t *rpc) +ds_pool_list_cont_handler(crt_rpc_t *rpc) { - ds_pool_list_cont_handler(rpc, 5); + pool_list_cont_handler(rpc, DAOS_POOL_VERSION); } /* TODO: consider moving to common function for client and engine use */ @@ -4824,7 +4799,7 @@ pool_cont_filter_is_valid(uuid_t pool_uuid, daos_pool_cont_filter_t *filt) * Requires a pool handle. */ static void -ds_pool_filter_cont_handler(crt_rpc_t *rpc, int handler_version) +pool_filter_cont_handler(crt_rpc_t *rpc, int handler_version) { struct pool_filter_cont_in *in = crt_req_get(rpc); struct pool_filter_cont_out *out = crt_reply_get(rpc); @@ -4922,19 +4897,13 @@ ds_pool_filter_cont_handler(crt_rpc_t *rpc, int handler_version) } void -ds_pool_filter_cont_handler_v6(crt_rpc_t *rpc) +ds_pool_filter_cont_handler(crt_rpc_t *rpc) { - ds_pool_filter_cont_handler(rpc, 6); -} - -void -ds_pool_filter_cont_handler_v5(crt_rpc_t *rpc) -{ - ds_pool_filter_cont_handler(rpc, 5); + pool_filter_cont_handler(rpc, DAOS_POOL_VERSION); } static void -ds_pool_query_handler(crt_rpc_t *rpc, int handler_version) +pool_query_handler(crt_rpc_t *rpc, int handler_version) { struct pool_query_in *in = crt_req_get(rpc); struct pool_query_out *out = crt_reply_get(rpc); @@ -5124,8 +5093,10 @@ ds_pool_query_handler(crt_rpc_t *rpc, int handler_version) /* See comment above, rebuild doesn't connect the pool */ if (query_bits & DAOS_PO_QUERY_SPACE) { - rc = pool_space_query_bcast(rpc->cr_ctx, svc, in->pqi_op.pi_hdl, - &out->pqo_space); + uint64_t *mem_file_bytes = handler_version >= 7 ? &out->pqo_mem_file_bytes : NULL; + + rc = pool_space_query_bcast(rpc->cr_ctx, svc, in->pqi_op.pi_hdl, &out->pqo_space, + mem_file_bytes); if (unlikely(rc)) goto out_svc; @@ -5150,15 +5121,15 @@ ds_pool_query_handler(crt_rpc_t *rpc, int handler_version) } void -ds_pool_query_handler_v5(crt_rpc_t *rpc) +ds_pool_query_handler_v6(crt_rpc_t *rpc) { - ds_pool_query_handler(rpc, 5); + pool_query_handler(rpc, 6); } void -ds_pool_query_handler_v6(crt_rpc_t *rpc) +ds_pool_query_handler(crt_rpc_t *rpc) { - ds_pool_query_handler(rpc, 6); + pool_query_handler(rpc, DAOS_POOL_VERSION); } /* Convert pool_comp_state_t to daos_target_state_t */ @@ -5179,8 +5150,8 @@ enum_pool_comp_state_to_tgt_state(int tgt_state) } static int -pool_query_tgt_space(crt_context_t ctx, struct pool_svc *svc, uuid_t pool_hdl, - d_rank_t rank, uint32_t tgt_idx, struct daos_space *ds) +pool_query_tgt_space(crt_context_t ctx, struct pool_svc *svc, uuid_t pool_hdl, d_rank_t rank, + uint32_t tgt_idx, struct daos_space *ds, uint64_t *mem_file_bytes) { struct pool_tgt_query_in *in; struct pool_tgt_query_out *out; @@ -5210,23 +5181,24 @@ pool_query_tgt_space(crt_context_t ctx, struct pool_svc *svc, uuid_t pool_hdl, if (rc != 0) goto out_rpc; + D_ASSERT(ds != NULL); out = crt_reply_get(rpc); - rc = out->tqo_rc; - if (rc != 0) { - D_ERROR(DF_UUID": failed to query rank:%u, tgt:%u, "DF_RC"\n", - DP_UUID(svc->ps_uuid), rank, tgt_idx, DP_RC(rc)); - } else { - D_ASSERT(ds != NULL); + rc = out->tqo_rc; + if (rc == 0) { *ds = out->tqo_space.ps_space; + if (mem_file_bytes != NULL) + *mem_file_bytes = out->tqo_mem_file_bytes; + } else { + D_ERROR(DF_UUID ": failed to query rank:%u, tgt:%u, " DF_RC "\n", + DP_UUID(svc->ps_uuid), rank, tgt_idx, DP_RC(rc)); } - out_rpc: crt_req_decref(rpc); return rc; } static void -ds_pool_query_info_handler(crt_rpc_t *rpc, int handler_version) +pool_query_info_handler(crt_rpc_t *rpc, int handler_version) { struct pool_query_info_in *in = crt_req_get(rpc); struct pool_query_info_out *out = crt_reply_get(rpc); @@ -5263,14 +5235,15 @@ ds_pool_query_info_handler(crt_rpc_t *rpc, int handler_version) tgt_state = target->ta_comp.co_status; out->pqio_state = enum_pool_comp_state_to_tgt_state(tgt_state); - out->pqio_op.po_map_version = - pool_map_get_version(svc->ps_pool->sp_map); + out->pqio_op.po_map_version = pool_map_get_version(svc->ps_pool->sp_map); ABT_rwlock_unlock(svc->ps_pool->sp_lock); if (tgt_state == PO_COMP_ST_UPIN) { + uint64_t *mem_file_bytes = handler_version >= 7 ? &out->pqio_mem_file_bytes : NULL; + rc = pool_query_tgt_space(rpc->cr_ctx, svc, in->pqii_op.pi_hdl, rank, tgt, - &out->pqio_space); + &out->pqio_space, mem_file_bytes); if (rc) DL_ERROR(rc, DF_UUID ": Failed to query rank:%u, tgt:%d", DP_UUID(in->pqii_op.pi_uuid), rank, tgt); @@ -5293,13 +5266,13 @@ ds_pool_query_info_handler(crt_rpc_t *rpc, int handler_version) void ds_pool_query_info_handler_v6(crt_rpc_t *rpc) { - ds_pool_query_info_handler(rpc, 6); + pool_query_info_handler(rpc, 6); } void -ds_pool_query_info_handler_v5(crt_rpc_t *rpc) +ds_pool_query_info_handler(crt_rpc_t *rpc) { - ds_pool_query_info_handler(rpc, 5); + pool_query_info_handler(rpc, DAOS_POOL_VERSION); } /** @@ -7416,7 +7389,7 @@ pool_discard(crt_context_t ctx, struct pool_svc *svc, struct pool_target_addr_li } static void -ds_pool_update_handler(crt_rpc_t *rpc, int handler_version) +pool_update_handler(crt_rpc_t *rpc, int handler_version) { struct pool_tgt_update_in *in = crt_req_get(rpc); struct pool_tgt_update_out *out = crt_reply_get(rpc); @@ -7466,15 +7439,9 @@ ds_pool_update_handler(crt_rpc_t *rpc, int handler_version) } void -ds_pool_update_handler_v6(crt_rpc_t *rpc) +ds_pool_update_handler(crt_rpc_t *rpc) { - ds_pool_update_handler(rpc, 6); -} - -void -ds_pool_update_handler_v5(crt_rpc_t *rpc) -{ - ds_pool_update_handler(rpc, 5); + pool_update_handler(rpc, DAOS_POOL_VERSION); } static int @@ -7972,7 +7939,7 @@ ds_pool_ranks_get_handler(crt_rpc_t *rpc) /* This RPC could be implemented by ds_rsvc. */ static void -ds_pool_svc_stop_handler(crt_rpc_t *rpc, int handler_version) +pool_svc_stop_handler(crt_rpc_t *rpc, int handler_version) { struct pool_svc_stop_in *in = crt_req_get(rpc); struct pool_svc_stop_out *out = crt_reply_get(rpc); @@ -7992,15 +7959,9 @@ ds_pool_svc_stop_handler(crt_rpc_t *rpc, int handler_version) } void -ds_pool_svc_stop_handler_v6(crt_rpc_t *rpc) -{ - ds_pool_svc_stop_handler(rpc, 6); -} - -void -ds_pool_svc_stop_handler_v5(crt_rpc_t *rpc) +ds_pool_svc_stop_handler(crt_rpc_t *rpc) { - ds_pool_svc_stop_handler(rpc, 5); + pool_svc_stop_handler(rpc, DAOS_POOL_VERSION); } /** @@ -8083,7 +8044,7 @@ ds_pool_svc_term_get(uuid_t uuid, uint64_t *term) } static void -ds_pool_attr_set_handler(crt_rpc_t *rpc, int handler_version) +pool_attr_set_handler(crt_rpc_t *rpc, int handler_version) { struct pool_attr_set_in *in = crt_req_get(rpc); struct pool_op_out *out = crt_reply_get(rpc); @@ -8159,19 +8120,13 @@ ds_pool_attr_set_handler(crt_rpc_t *rpc, int handler_version) } void -ds_pool_attr_set_handler_v6(crt_rpc_t *rpc) +ds_pool_attr_set_handler(crt_rpc_t *rpc) { - ds_pool_attr_set_handler(rpc, 6); -} - -void -ds_pool_attr_set_handler_v5(crt_rpc_t *rpc) -{ - ds_pool_attr_set_handler(rpc, 5); + pool_attr_set_handler(rpc, DAOS_POOL_VERSION); } static void -ds_pool_attr_del_handler(crt_rpc_t *rpc, int handler_version) +pool_attr_del_handler(crt_rpc_t *rpc, int handler_version) { struct pool_attr_del_in *in = crt_req_get(rpc); struct pool_op_out *out = crt_reply_get(rpc); @@ -8246,19 +8201,13 @@ ds_pool_attr_del_handler(crt_rpc_t *rpc, int handler_version) } void -ds_pool_attr_del_handler_v6(crt_rpc_t *rpc) -{ - ds_pool_attr_del_handler(rpc, 6); -} - -void -ds_pool_attr_del_handler_v5(crt_rpc_t *rpc) +ds_pool_attr_del_handler(crt_rpc_t *rpc) { - ds_pool_attr_del_handler(rpc, 5); + pool_attr_del_handler(rpc, DAOS_POOL_VERSION); } static void -ds_pool_attr_get_handler(crt_rpc_t *rpc, int handler_version) +pool_attr_get_handler(crt_rpc_t *rpc, int handler_version) { struct pool_attr_get_in *in = crt_req_get(rpc); struct pool_op_out *out = crt_reply_get(rpc); @@ -8297,19 +8246,13 @@ ds_pool_attr_get_handler(crt_rpc_t *rpc, int handler_version) } void -ds_pool_attr_get_handler_v6(crt_rpc_t *rpc) +ds_pool_attr_get_handler(crt_rpc_t *rpc) { - ds_pool_attr_get_handler(rpc, 6); -} - -void -ds_pool_attr_get_handler_v5(crt_rpc_t *rpc) -{ - ds_pool_attr_get_handler(rpc, 5); + pool_attr_get_handler(rpc, DAOS_POOL_VERSION); } static void -ds_pool_attr_list_handler(crt_rpc_t *rpc, int handler_version) +pool_attr_list_handler(crt_rpc_t *rpc, int handler_version) { struct pool_attr_list_in *in = crt_req_get(rpc); struct pool_attr_list_out *out = crt_reply_get(rpc); @@ -8347,15 +8290,9 @@ ds_pool_attr_list_handler(crt_rpc_t *rpc, int handler_version) } void -ds_pool_attr_list_handler_v6(crt_rpc_t *rpc) -{ - ds_pool_attr_list_handler(rpc, 6); -} - -void -ds_pool_attr_list_handler_v5(crt_rpc_t *rpc) +ds_pool_attr_list_handler(crt_rpc_t *rpc) { - ds_pool_attr_list_handler(rpc, 5); + pool_attr_list_handler(rpc, DAOS_POOL_VERSION); } void diff --git a/src/pool/srv_target.c b/src/pool/srv_target.c index 5d19f784d79..8d1cc7d6aa4 100644 --- a/src/pool/srv_target.c +++ b/src/pool/srv_target.c @@ -1476,8 +1476,8 @@ ds_pool_hdl_put(struct ds_pool_hdl *hdl) } static void -aggregate_pool_space(struct daos_pool_space *agg_ps, - struct daos_pool_space *ps) +aggregate_pool_space(struct daos_pool_space *agg_ps, uint64_t *agg_mem_bytes, + struct daos_pool_space *ps, uint64_t *mem_bytes) { int i; bool first; @@ -1504,11 +1504,16 @@ aggregate_pool_space(struct daos_pool_space *agg_ps, agg_ps->ps_free_mean[i] = agg_ps->ps_space.s_free[i] / agg_ps->ps_ntargets; } + if (agg_mem_bytes != NULL) { + D_ASSERT(mem_bytes != NULL); + *agg_mem_bytes += *mem_bytes; + } } struct pool_query_xs_arg { struct ds_pool *qxa_pool; struct daos_pool_space qxa_space; + uint64_t qxa_mem_bytes; }; static void @@ -1521,7 +1526,8 @@ pool_query_xs_reduce(void *agg_arg, void *xs_arg) return; D_ASSERT(x_arg->qxa_space.ps_ntargets == 1); - aggregate_pool_space(&a_arg->qxa_space, &x_arg->qxa_space); + aggregate_pool_space(&a_arg->qxa_space, &a_arg->qxa_mem_bytes, &x_arg->qxa_space, + &x_arg->qxa_mem_bytes); } static int @@ -1546,7 +1552,7 @@ pool_query_xs_arg_free(struct dss_stream_arg_type *xs) } static int -pool_query_space(uuid_t pool_uuid, struct daos_pool_space *x_ps) +pool_query_space(uuid_t pool_uuid, struct daos_pool_space *x_ps, uint64_t *mem_file_bytes) { struct dss_module_info *info = dss_get_module_info(); int tid = info->dmi_tgt_id; @@ -1570,6 +1576,8 @@ pool_query_space(uuid_t pool_uuid, struct daos_pool_space *x_ps) x_ps->ps_ntargets = 1; x_ps->ps_space.s_total[DAOS_MEDIA_SCM] = SCM_TOTAL(vps); x_ps->ps_space.s_total[DAOS_MEDIA_NVME] = NVME_TOTAL(vps); + if (mem_file_bytes != NULL) + *mem_file_bytes = vps->vps_mem_bytes; /* Exclude the sys reserved space before reporting to user */ if (SCM_FREE(vps) > SCM_SYS(vps)) @@ -1603,11 +1611,11 @@ pool_query_one(void *vin) struct pool_query_xs_arg *x_arg = streams[tid].st_arg; struct ds_pool *pool = x_arg->qxa_pool; - return pool_query_space(pool->sp_uuid, &x_arg->qxa_space); + return pool_query_space(pool->sp_uuid, &x_arg->qxa_space, &x_arg->qxa_mem_bytes); } static int -pool_tgt_query(struct ds_pool *pool, struct daos_pool_space *ps) +pool_tgt_query(struct ds_pool *pool, struct daos_pool_space *ps, uint64_t *mem_file_bytes) { struct dss_coll_ops coll_ops; struct dss_coll_args coll_args = { 0 }; @@ -1640,6 +1648,8 @@ pool_tgt_query(struct ds_pool *pool, struct daos_pool_space *ps) } *ps = agg_arg.qxa_space; + if (mem_file_bytes != NULL) + *mem_file_bytes = agg_arg.qxa_mem_bytes; out: return rc; @@ -1952,17 +1962,23 @@ ds_pool_tgt_map_update(struct ds_pool *pool, struct pool_buf *buf, return rc; } -void -ds_pool_tgt_query_handler(crt_rpc_t *rpc) +static void +pool_tgt_query_handler(crt_rpc_t *rpc, int handler_version) { struct pool_tgt_query_in *in = crt_req_get(rpc); struct pool_tgt_query_out *out = crt_reply_get(rpc); struct ds_pool *pool; + uint64_t *mem_file_bytes; int rc; + if (handler_version >= 7) + mem_file_bytes = &out->tqo_mem_file_bytes; + else + mem_file_bytes = NULL; + /* Single target query */ if (dss_get_module_info()->dmi_xs_id != 0) { - rc = pool_query_space(in->tqi_op.pi_uuid, &out->tqo_space); + rc = pool_query_space(in->tqi_op.pi_uuid, &out->tqo_space, mem_file_bytes); goto out; } @@ -1974,7 +1990,7 @@ ds_pool_tgt_query_handler(crt_rpc_t *rpc) D_GOTO(out, rc = -DER_NONEXIST); } - rc = pool_tgt_query(pool, &out->tqo_space); + rc = pool_tgt_query(pool, &out->tqo_space, mem_file_bytes); if (rc != 0) rc = 1; /* For query aggregator */ ds_pool_put(pool); @@ -1983,6 +1999,32 @@ ds_pool_tgt_query_handler(crt_rpc_t *rpc) crt_reply_send(rpc); } +void +ds_pool_tgt_query_handler_v6(crt_rpc_t *rpc) +{ + pool_tgt_query_handler(rpc, 6); +} + +void +ds_pool_tgt_query_handler(crt_rpc_t *rpc) +{ + pool_tgt_query_handler(rpc, DAOS_POOL_VERSION); +} + +int +ds_pool_tgt_query_aggregator_v6(crt_rpc_t *source, crt_rpc_t *result, void *priv) +{ + struct pool_tgt_query_v6_out *out_source = crt_reply_get(source); + struct pool_tgt_query_v6_out *out_result = crt_reply_get(result); + + out_result->tqo_rc += out_source->tqo_rc; + if (out_source->tqo_rc != 0) + return 0; + + aggregate_pool_space(&out_result->tqo_space, NULL, &out_source->tqo_space, NULL); + return 0; +} + int ds_pool_tgt_query_aggregator(crt_rpc_t *source, crt_rpc_t *result, void *priv) { @@ -1993,7 +2035,8 @@ ds_pool_tgt_query_aggregator(crt_rpc_t *source, crt_rpc_t *result, void *priv) if (out_source->tqo_rc != 0) return 0; - aggregate_pool_space(&out_result->tqo_space, &out_source->tqo_space); + aggregate_pool_space(&out_result->tqo_space, &out_result->tqo_mem_file_bytes, + &out_source->tqo_space, &out_source->tqo_mem_file_bytes); return 0; } diff --git a/src/vos/vos_layout.h b/src/vos/vos_layout.h index 87d092bc882..79fc7cbc7ae 100644 --- a/src/vos/vos_layout.h +++ b/src/vos/vos_layout.h @@ -119,8 +119,10 @@ struct vos_gc_bkt_df { struct vos_pool_ext_df { /* Extension for GC bucket */ struct vos_gc_bkt_df ped_gc_bkt; + /* Memory file size for md-on-ssd phase2 pool */ + uint64_t ped_mem_sz; /* Paddings for other potential new feature */ - uint64_t ped_paddings[54]; + uint64_t ped_paddings[53]; /* Reserved for future extension */ uint64_t ped_reserve; }; diff --git a/src/vos/vos_pool.c b/src/vos/vos_pool.c index dcf11cfdcca..ba04cd6ae20 100644 --- a/src/vos/vos_pool.c +++ b/src/vos/vos_pool.c @@ -1226,6 +1226,7 @@ vos_pool_create_ex(const char *path, uuid_t uuid, daos_size_t scm_sz, daos_size_ daos_handle_t hdl; struct d_uuid ukey; struct vos_pool *pool = NULL; + struct vos_pool_ext_df *pd_ext_df; int rc = 0; if (!path || uuid_is_null(uuid) || daos_file_is_dax(path)) @@ -1326,6 +1327,10 @@ vos_pool_create_ex(const char *path, uuid_t uuid, daos_size_t scm_sz, daos_size_ pool_df->pd_version = 0; else pool_df->pd_version = version; + + /* pd_ext is newly allocated, no need to call tx_add_ptr() */ + pd_ext_df = umem_off2ptr(&umem, pool_df->pd_ext); + pd_ext_df->ped_mem_sz = scm_sz; end: /** * The transaction can in reality be aborted diff --git a/src/vos/vos_space.c b/src/vos/vos_space.c index 35a407e2b3e..132253c1cb4 100644 --- a/src/vos/vos_space.c +++ b/src/vos/vos_space.c @@ -145,6 +145,11 @@ vos_space_query(struct vos_pool *pool, struct vos_pool_space *vps, bool slow) /* Query non-evictable zones usage when the phase2 pool is evictable */ if (vos_pool_is_evictable(pool)) { + struct vos_pool_ext_df *pd_ext_df = umem_off2ptr(vos_pool2umm(pool), df->pd_ext); + + D_ASSERT(pd_ext_df != NULL); + vps->vps_mem_bytes = pd_ext_df->ped_mem_sz; + rc = umempobj_get_mbusage(vos_pool2umm(pool)->umm_pool, UMEM_DEFAULT_MBKT_ID, &ne_used, &vps->vps_ne_total); if (rc) { @@ -160,6 +165,7 @@ vos_space_query(struct vos_pool *pool, struct vos_pool_space *vps, bool slow) } vps->vps_ne_free = vps->vps_ne_total - ne_used; } else { + vps->vps_mem_bytes = SCM_TOTAL(vps); vps->vps_ne_total = 0; vps->vps_ne_free = 0; } From b31f634bcf4549a6fca5b925add7e1a524b99f3c Mon Sep 17 00:00:00 2001 From: Dalton Bohning Date: Tue, 17 Dec 2024 08:39:26 -0800 Subject: [PATCH 08/10] DAOS-16889 build: fix finding protobuf (#15625) Add a requirement to protobufc for building daos control binaries. Signed-off-by: Dalton Bohning --- src/control/SConscript | 1 + 1 file changed, 1 insertion(+) diff --git a/src/control/SConscript b/src/control/SConscript index 17b654f162d..e70247b4c7d 100644 --- a/src/control/SConscript +++ b/src/control/SConscript @@ -140,6 +140,7 @@ def scons(): # Sets CGO_LDFLAGS for rpath options denv.d_add_rpaths("..", True, True) + denv.require('protobufc') denv.AppendENVPath("CGO_CFLAGS", denv.subst("$_CPPINCFLAGS"), sep=" ") if prereqs.client_requested(): install_go_bin(denv, "daos_agent") From b020347a79f9fb01381b862786117f78ce0b7c97 Mon Sep 17 00:00:00 2001 From: Michael MacDonald Date: Tue, 17 Dec 2024 15:45:55 -0500 Subject: [PATCH 09/10] DAOS-16840 control: Deprecate access_points in server config (#15548) The access_points name in the server configuration is an ongoing source of confusion. Rename it to the more descriptive mgmt_svc_replicas and emit deprecation notices for older configurations. Signed-off-by: Michael MacDonald --- docs/QSG/qemu-vms.md | 4 +- docs/QSG/setup_rhel.md | 2 +- docs/QSG/setup_suse.md | 2 +- docs/QSG/tour.md | 2 +- docs/admin/administration.md | 20 +-- docs/admin/common_tasks.md | 4 +- docs/admin/deployment.md | 24 ++-- docs/admin/troubleshooting.md | 6 +- src/control/cmd/daos_server/README.md | 2 +- src/control/cmd/daos_server/auto.go | 2 + src/control/cmd/daos_server/auto_test.go | 77 ++++++----- src/control/cmd/daos_server/config.go | 7 +- src/control/cmd/daos_server/main.go | 2 +- src/control/cmd/daos_server/start_test.go | 1 + src/control/cmd/dmg/auto.go | 4 +- src/control/cmd/dmg/auto_test.go | 53 ++++---- src/control/common/cmdutil/auto.go | 18 ++- src/control/fault/code/codes.go | 4 +- src/control/lib/control/auto.go | 36 ++--- src/control/lib/control/auto_test.go | 49 +++---- src/control/lib/control/event.go | 24 ++-- src/control/lib/control/event_test.go | 14 +- src/control/lib/support/log.go | 8 +- src/control/server/config/faults.go | 18 +-- src/control/server/config/server.go | 78 ++++++----- src/control/server/config/server_test.go | 127 +++++++++--------- src/control/server/mgmt_pool_test.go | 49 +++---- src/control/server/mgmt_system.go | 2 +- src/control/server/server.go | 4 +- src/control/server/server_utils.go | 8 +- src/control/server/server_utils_test.go | 2 +- src/control/server/util_test.go | 2 +- src/tests/ftest/config_file_gen.py | 12 +- .../ftest/control/config_generate_output.py | 83 ++++++------ .../ftest/control/config_generate_run.py | 2 +- .../ftest/control/config_generate_run.yaml | 2 +- .../ftest/control/daos_control_config.py | 4 +- src/tests/ftest/control/daos_system_query.py | 8 +- src/tests/ftest/control/ms_failover.py | 10 +- src/tests/ftest/control/ms_resilience.py | 8 +- src/tests/ftest/harness/config.py | 40 +++--- src/tests/ftest/harness/config.yaml | 2 +- src/tests/ftest/network/cart_self_test.py | 2 +- src/tests/ftest/pool/destroy.py | 6 +- src/tests/ftest/pool/destroy_rebuild.py | 26 ++-- src/tests/ftest/pool/destroy_rebuild.yaml | 2 +- .../ftest/server/daos_server_config.yaml | 16 +-- src/tests/ftest/util/agent_utils_params.py | 5 + src/tests/ftest/util/apricot/apricot/test.py | 102 +++++++------- src/tests/ftest/util/command_utils_base.py | 7 - src/tests/ftest/util/dmg_utils.py | 6 +- src/tests/ftest/util/dmg_utils_base.py | 2 +- src/tests/ftest/util/host_utils.py | 6 +- src/tests/ftest/util/server_utils.py | 11 +- src/tests/ftest/util/server_utils_params.py | 5 + src/tests/ftest/util/test_utils_pool.py | 2 +- utils/config/daos_server.yml | 14 +- utils/config/examples/daos_server_local.yml | 2 +- utils/config/examples/daos_server_mdonssd.yml | 9 +- utils/config/examples/daos_server_tcp.yml | 9 +- utils/config/examples/daos_server_ucx.yml | 9 +- utils/config/examples/daos_server_verbs.yml | 9 +- .../daos-server/el8/daos_server.yml.example | 2 +- .../daos-server/el8/daos_server.yml.in | 2 +- utils/nlt_server.yaml | 2 +- utils/node_local_test.py | 2 +- 66 files changed, 582 insertions(+), 502 deletions(-) diff --git a/docs/QSG/qemu-vms.md b/docs/QSG/qemu-vms.md index 108d89295d9..3531a82fb3a 100644 --- a/docs/QSG/qemu-vms.md +++ b/docs/QSG/qemu-vms.md @@ -199,10 +199,10 @@ I follow these [steps](https://docs.daos.io/latest/QSG/setup_rhel/) to install b 5. Update config files. -Update the daos-server config file `/etc/daos/daos_server.yml` on daos-server. You may need to update "access\_points", "fabric\_iface" and "bdev\_list". Update "access\_points" accordingly if you name daos-server differently. Check if the network device has the same name as listed under "fabric\_iface". Look in the output of `lspci` for "bdev\_list". The info for our NVMe controller is like *??:??:? Non-Volatile memory controller: Red Hat, Inc. QEMU NVM Express Controller (rev 02)*. Prefix *??:??.?* is the address of the NVMe devices. +Update the daos-server config file `/etc/daos/daos_server.yml` on daos-server. You may need to update "mgmt\_svc\_replicas", "fabric\_iface" and "bdev\_list". Update "mgmt\_svc\_replicas" accordingly if you name daos-server differently. Check if the network device has the same name as listed under "fabric\_iface". Look in the output of `lspci` for "bdev\_list". The info for our NVMe controller is like *??:??:? Non-Volatile memory controller: Red Hat, Inc. QEMU NVM Express Controller (rev 02)*. Prefix *??:??.?* is the address of the NVMe devices. ``` name: daos_server -access_points: +mgmt_svc_replicas: - daos-server port: 10001 diff --git a/docs/QSG/setup_rhel.md b/docs/QSG/setup_rhel.md index 9ed73ce8ec8..dbe855ae762 100644 --- a/docs/QSG/setup_rhel.md +++ b/docs/QSG/setup_rhel.md @@ -273,7 +273,7 @@ Examples are available on [github](https://github.com/daos-stack/daos/tree/maste name: daos_server - access_points: + mgmt_svc_replicas: - server-1 port: 10001 diff --git a/docs/QSG/setup_suse.md b/docs/QSG/setup_suse.md index c05b30945ac..32500dfe4fb 100644 --- a/docs/QSG/setup_suse.md +++ b/docs/QSG/setup_suse.md @@ -292,7 +292,7 @@ Examples are available on [github](https://github.com/daos-stack/daos/tree/maste An example of the daos_server.yml is presented below. Copy the modified server yaml file to all the server nodes at `/etc/daos/daos_server.yml`. name: daos_server - access_points: + mgmt_svc_replicas: - node-4 port: 10001 diff --git a/docs/QSG/tour.md b/docs/QSG/tour.md index e3254ac25cd..78dd290cc03 100644 --- a/docs/QSG/tour.md +++ b/docs/QSG/tour.md @@ -223,7 +223,7 @@ bring-up DAOS servers and clients. ### Run dfuse # Bring up 4 hosts server with appropriate daos_server.yml and - # access-point, reference to DAOS Set-Up + # MS replicas, reference to DAOS Set-Up # After DAOS servers, DAOS admin and client started. $ dmg storage format diff --git a/docs/admin/administration.md b/docs/admin/administration.md index 9859baebd2a..049e98eb81c 100644 --- a/docs/admin/administration.md +++ b/docs/admin/administration.md @@ -825,9 +825,9 @@ device would remain in this state until replaced by a new device. ## System Operations -The DAOS server acting as the access point records details of engines -that join the DAOS system. Once an engine has joined the DAOS system, it is -identified by a unique system "rank". Multiple ranks can reside on the same +The DAOS server acting as the Management Service (MS) leader records details +of engines that join the DAOS system. Once an engine has joined the DAOS system, +it is identified by a unique system "rank". Multiple ranks can reside on the same host machine, accessible via the same network address. A DAOS system can be shutdown and restarted to perform maintenance and/or @@ -837,14 +837,14 @@ made to the rank's metadata stored on persistent memory. Storage reformat can also be performed after system shutdown. Pools will be removed and storage wiped. -System commands will be handled by a DAOS Server acting as access point and +System commands will be handled by a DAOS Server acting as the MS leader and listening on the address specified in the DMG config file "hostlist" parameter. See [`daos_control.yml`](https://github.com/daos-stack/daos/blob/master/utils/config/daos_control.yml) for details. At least one of the addresses in the hostlist parameters should match one of the -"access point" addresses specified in the server config file +`mgmt_svc_replicas` addresses specified in the server config file [`daos_server.yml`](https://github.com/daos-stack/daos/blob/master/utils/config/daos_server.yml) that is supplied when starting `daos_server` instances. @@ -1028,13 +1028,15 @@ formatted again by running `dmg storage format`. To add a new server to an existing DAOS system, one should install: -- the relevant certificates -- the server yaml file pointing to the access points of the running - DAOS system +- A copy of the relevant certificates from an existing server. All servers must + share the same set of certificates in order to provide services. +- A copy of the server yaml file from an existing server (DAOS server configurations + should be homogeneous) -- the `mgmt_svc_replicas` entry is used by the new server in + order to know which servers should handle its SystemJoin request. The daos\_control.yml file should also be updated to include the new DAOS server. -Then starts the daos\_server via systemd and format the new server via +Then start the daos\_server via systemd and format the new server via dmg as follows: ``` diff --git a/docs/admin/common_tasks.md b/docs/admin/common_tasks.md index 065edd13d11..fef3dbe9e60 100644 --- a/docs/admin/common_tasks.md +++ b/docs/admin/common_tasks.md @@ -9,7 +9,7 @@ This section describes some of the common tasks handled by admins at a high leve 3. Install `daos-server` and `daos-client` RPMs. 4. Generate certificate files. 5. Copy one of the example configs from `utils/config/examples` to -`/etc/daos` and adjust it based on the environment. E.g., `access_points`, +`/etc/daos` and adjust it based on the environment. E.g., `mgmt_svc_replicas`, `class`. 6. Check that the directory where the log files will be created exists. E.g., `control_log_file`, `log_file` field in `engines` section. @@ -38,7 +38,7 @@ to server hosts and `daos-client` to client hosts. 4. Generate certificate files and distribute them to all the hosts. 5. Copy one of the example configs from `utils/config/examples` to `/etc/daos` of one of the server hosts and adjust it based on the environment. -E.g., `access_points`, `class`. +E.g., `mgmt_svc_replicas`, `class`. 6. Check that the directory where the log files will be created exists. E.g., `control_log_file`, `log_file` field in `engines` section. 7. Start `daos_server`. diff --git a/docs/admin/deployment.md b/docs/admin/deployment.md index 0f5c128b1f4..e318397a28d 100644 --- a/docs/admin/deployment.md +++ b/docs/admin/deployment.md @@ -50,7 +50,7 @@ A recommended workflow to get up and running is as follows: server config file (default location at `/etc/daos/daos_server.yml`) has not yet been populated. -* Run `dmg config generate -l -a ` across the entire +* Run `dmg config generate -l -r ` across the entire hostset (all the storage servers that are now running the `daos_server` service after RPM install). The command will only generate a config if hardware setups on all the hosts are @@ -285,7 +285,7 @@ Help Options: [generate command options] -l, --helper-log-file= Log file location for debug from daos_server_helper binary - -a, --access-points= Comma separated list of access point addresses + -r, --ms-replicas= Comma separated list of MS replica addresses (default: localhost) -e, --num-engines= Set the number of DAOS Engine sections to be populated in the config file output. If unset then the value will be set to the @@ -331,7 +331,7 @@ Help Options: [generate command options] -l, --host-list= A comma separated list of addresses to connect to - -a, --access-points= Comma separated list of access point addresses + -r, --ms-replicas= Comma separated list of MS replica addresses to host management service (default: localhost) -e, --num-engines= Set the number of DAOS Engine sections to be populated in the config file output. If unset then the value will be set to the @@ -371,8 +371,8 @@ engines: The options that can be supplied to the config generate command are as follows: -- `--access-points` specifies the access points (identified storage servers that will host the -management service for the DAOS system across the cluster). +- `--ms-replicas` specifies the MS replicas (identified storage servers that will host the +Management Service for the DAOS system across the cluster). - `--num-engines` specifies the number of engine sections to populate in the config file output. If not set explicitly on the commandline, default is the number of NUMA nodes detected on the host. @@ -502,7 +502,7 @@ core_dump_filter: 19 name: daos_server socket_dir: /var/run/daos_server provider: ofi+tcp -access_points: +mgmt_svc_replicas: - localhost:10001 fault_cb: "" hyperthreads: false @@ -515,7 +515,7 @@ and runs until the point where a storage format is required, as expected. [user@wolf-226 daos]$ install/bin/daos_server start -i -o ~/configs/tmp.yml DAOS Server config loaded from /home/user/configs/tmp.yml install/bin/daos_server logging to file /tmp/daos_server.log -NOTICE: Configuration includes only one access point. This provides no redundancy in the event of an access point failure. +NOTICE: Configuration includes only one MS replica. This provides no redundancy in the event of a MS replica failure. DAOS Control Server v2.3.101 (pid 1211553) listening on 127.0.0.1:10001 Checking DAOS I/O Engine instance 0 storage ... Checking DAOS I/O Engine instance 1 storage ... @@ -821,8 +821,6 @@ To set the addresses of which DAOS Servers to task, provide either: Where `` represents a slurm-style hostlist string e.g. `foo-1[28-63],bar[256-511]`. -The first entry in the hostlist (after alphabetic then numeric sorting) will be -assumed to be the access point as set in the server configuration file. Local configuration files stored in the user directory will be used in preference to the default location e.g. `~/.daos_control.yml`. @@ -1322,7 +1320,7 @@ as follows to establish 2-tier storage: ```yaml port: 10001 -access_points: ["wolf-71"] # <----- updated +mgmt_svc_replicas: ["wolf-71"] # <----- updated engines: - @@ -1367,10 +1365,10 @@ information, please refer to the [DAOS build documentation][6]. DAOS Control Servers will need to be restarted on all hosts after updates to the server configuration file. - Pick an odd number of hosts in the system and set `access_points` to list of that host's - hostname or IP address (don't need to specify port). + Pick an odd number (3-7) of hosts in the system and set the `mgmt_svc_replicas` list to + include the hostnames or IP addresses (don't need to specify port) of those hosts. - This will be the host which bootstraps the DAOS management service (MS). + This will be the set of servers which host the replicated DAOS management service (MS). >The support of the optional providers is not guarantee and can be removed >without further notification. diff --git a/docs/admin/troubleshooting.md b/docs/admin/troubleshooting.md index 79173e782f7..5de7b95412d 100644 --- a/docs/admin/troubleshooting.md +++ b/docs/admin/troubleshooting.md @@ -313,7 +313,7 @@ sudo ipcrm -M 0x10242049 1. Format the SCMs defined in the config file. 1. Generate the config file using `dmg config generate`. The various requirements will be populated without a syntax error. 1. Try starting with `allow_insecure: true`. This will rule out the credential certificate issue. -1. Verify that the `access_points` host is accessible and the port is not used. +1. Verify that the `mgmt_svc_replicas` host is accessible and the port is not used. 1. Check the `provider` entry. See the "Network Scan and Configuration" section of the admin guide for determining the right provider to use. 1. Check `fabric_iface` in `engines`. They should be available and enabled. 1. Check that `socket_dir` is writable by the daos_server. @@ -327,7 +327,7 @@ sudo ipcrm -M 0x10242049 1. When the server configuration is changed, it's necessary to restart the agent. 1. `DER_UNREACH(-1006)`: Check the socket ID consistency between PMem and NVMe. First, determine which socket you're using with `daos_server network scan -p all`. e.g., if the interface you're using in the engine section is eth0, find which NUMA Socket it belongs to. Next, determine the disks you can use with this socket by calling `daos_server nvme scan` or `dmg storage scan`. e.g., if eth0 belongs to NUMA Socket 0, use only the disks with 0 in the Socket ID column. 1. Check the interface used in the server config (`fabric_iface`) also exists in the client and can communicate with the server. -1. Check the access_points of the agent config points to the correct server host. +1. Check the `access_points` of the agent config points to the correct server hosts. 1. Call `daos pool query` and check that the pool exists and has free space. ### Applications run slow @@ -512,7 +512,7 @@ fabric providers. After starting `daos_server`, ranks will be unable to join if their configuration's fabric provider does not match that of the system. The system configuration is determined by the management service -(MS) leader node, which may be arbitrarily chosen from the configured access points. +(MS) leader node, which may be arbitrarily chosen from the configured MS replicas. The error message will include the string: `fabric provider does not match system provider ` diff --git a/src/control/cmd/daos_server/README.md b/src/control/cmd/daos_server/README.md index 86f652faed8..bd8be525790 100644 --- a/src/control/cmd/daos_server/README.md +++ b/src/control/cmd/daos_server/README.md @@ -120,7 +120,7 @@ The control API is responsible for working out which `daos_server` instance is the MS leader and issuing the request, `dmg` uses the control API. The `dmg` tool requires the hostlist of all hosts in the DAOS system to be specified either on the command line or in the `daos_control.yml` config file. -A list of access point servers is defined in the server's config file. +A list of MS replica servers is defined in the server's config file. ## Functionality diff --git a/src/control/cmd/daos_server/auto.go b/src/control/cmd/daos_server/auto.go index d1c28a3f5d0..b2f7f690b6c 100644 --- a/src/control/cmd/daos_server/auto.go +++ b/src/control/cmd/daos_server/auto.go @@ -122,6 +122,8 @@ func (cmd *configGenCmd) confGen(ctx context.Context, getFabric getFabricFn, get } cmd.Debugf("fetched host storage info on localhost: %+v", hs) + cmd.CheckDeprecated(cmd.Logger) + req := new(control.ConfGenerateReq) if err := convert.Types(cmd, req); err != nil { return nil, err diff --git a/src/control/cmd/daos_server/auto_test.go b/src/control/cmd/daos_server/auto_test.go index 8fb8d85522f..2008c41c30b 100644 --- a/src/control/cmd/daos_server/auto_test.go +++ b/src/control/cmd/daos_server/auto_test.go @@ -32,11 +32,11 @@ import ( func TestDaosServer_Auto_Commands(t *testing.T) { runCmdTests(t, []cmdTest{ { - "Generate with no access point", + "Generate with no MS replica", "config generate", printCommand(t, func() *configGenCmd { cmd := &configGenCmd{} - cmd.AccessPoints = "localhost" + cmd.MgmtSvcReplicas = "localhost" cmd.NetClass = "infiniband" return cmd }()), @@ -44,10 +44,10 @@ func TestDaosServer_Auto_Commands(t *testing.T) { }, { "Generate with defaults", - "config generate -a foo", + "config generate -r foo", printCommand(t, func() *configGenCmd { cmd := &configGenCmd{} - cmd.AccessPoints = "foo" + cmd.MgmtSvcReplicas = "foo" cmd.NetClass = "infiniband" return cmd }()), @@ -55,10 +55,10 @@ func TestDaosServer_Auto_Commands(t *testing.T) { }, { "Generate with no nvme", - "config generate -a foo --scm-only", + "config generate -r foo --scm-only", printCommand(t, func() *configGenCmd { cmd := &configGenCmd{} - cmd.AccessPoints = "foo" + cmd.MgmtSvcReplicas = "foo" cmd.NetClass = "infiniband" cmd.SCMOnly = true return cmd @@ -67,10 +67,10 @@ func TestDaosServer_Auto_Commands(t *testing.T) { }, { "Generate with storage parameters", - "config generate -a foo --num-engines 2", + "config generate -r foo --num-engines 2", printCommand(t, func() *configGenCmd { cmd := &configGenCmd{} - cmd.AccessPoints = "foo" + cmd.MgmtSvcReplicas = "foo" cmd.NetClass = "infiniband" cmd.NrEngines = 2 return cmd @@ -79,10 +79,10 @@ func TestDaosServer_Auto_Commands(t *testing.T) { }, { "Generate with short option storage parameters", - "config generate -a foo -e 2 -s", + "config generate -r foo -e 2 -s", printCommand(t, func() *configGenCmd { cmd := &configGenCmd{} - cmd.AccessPoints = "foo" + cmd.MgmtSvcReplicas = "foo" cmd.NetClass = "infiniband" cmd.NrEngines = 2 cmd.SCMOnly = true @@ -92,10 +92,10 @@ func TestDaosServer_Auto_Commands(t *testing.T) { }, { "Generate with ethernet network device class", - "config generate -a foo --net-class ethernet", + "config generate -r foo --net-class ethernet", printCommand(t, func() *configGenCmd { cmd := &configGenCmd{} - cmd.AccessPoints = "foo" + cmd.MgmtSvcReplicas = "foo" cmd.NetClass = "ethernet" return cmd }()), @@ -103,10 +103,10 @@ func TestDaosServer_Auto_Commands(t *testing.T) { }, { "Generate with infiniband network device class", - "config generate -a foo --net-class infiniband", + "config generate -r foo --net-class infiniband", printCommand(t, func() *configGenCmd { cmd := &configGenCmd{} - cmd.AccessPoints = "foo" + cmd.MgmtSvcReplicas = "foo" cmd.NetClass = "infiniband" return cmd }()), @@ -114,22 +114,22 @@ func TestDaosServer_Auto_Commands(t *testing.T) { }, { "Generate with deprecated network device class", - "config generate -a foo --net-class best-available", + "config generate -r foo --net-class best-available", "", errors.New("Invalid value"), }, { "Generate with unsupported network device class", - "config generate -a foo --net-class loopback", + "config generate -r foo --net-class loopback", "", errors.New("Invalid value"), }, { "Generate tmpfs non-MD-on-SSD config", - "config generate -a foo --use-tmpfs-scm", + "config generate -r foo --use-tmpfs-scm", printCommand(t, func() *configGenCmd { cmd := &configGenCmd{} - cmd.AccessPoints = "foo" + cmd.MgmtSvcReplicas = "foo" cmd.NetClass = "infiniband" cmd.UseTmpfsSCM = true return cmd @@ -138,10 +138,10 @@ func TestDaosServer_Auto_Commands(t *testing.T) { }, { "Generate MD-on-SSD config", - "config generate -a foo --use-tmpfs-scm --control-metadata-path /opt/daos_md", + "config generate -r foo --use-tmpfs-scm --control-metadata-path /opt/daos_md", printCommand(t, func() *configGenCmd { cmd := &configGenCmd{} - cmd.AccessPoints = "foo" + cmd.MgmtSvcReplicas = "foo" cmd.NetClass = "infiniband" cmd.UseTmpfsSCM = true cmd.ExtMetadataPath = "/opt/daos_md" @@ -169,7 +169,7 @@ func TestDaosServer_Auto_confGenCmd_Convert(t *testing.T) { cmd.NrEngines = 1 cmd.NetProvider = "ofi+tcp" cmd.SCMOnly = true - cmd.AccessPoints = "foo,bar" + cmd.MgmtSvcReplicas = "foo,bar" cmd.NetClass = "infiniband" cmd.UseTmpfsSCM = true cmd.ExtMetadataPath = "/opt/daos_md" @@ -184,7 +184,7 @@ func TestDaosServer_Auto_confGenCmd_Convert(t *testing.T) { NrEngines: 1, NetProvider: "ofi+tcp", SCMOnly: true, - AccessPoints: []string{"foo", "bar"}, + MgmtSvcReplicas: []string{"foo", "bar"}, NetClass: hardware.Infiniband, UseTmpfsSCM: true, ExtMetadataPath: "/opt/daos_md", @@ -273,7 +273,7 @@ func TestDaosServer_Auto_confGen(t *testing.T) { } for name, tc := range map[string]struct { - accessPoints string + msReplicas string nrEngines int scmOnly bool netClass string @@ -338,16 +338,16 @@ func TestDaosServer_Auto_confGen(t *testing.T) { hf: defHostFabric, hs: defHostStorage, expCfg: control.MockServerCfg("ofi+psm2", exmplEngineCfgs). - WithAccessPoints("localhost:10001"). + WithMgmtSvcReplicas("localhost:10001"). WithControlLogFile("/tmp/daos_server.log"), }, - "access points set": { - accessPoints: "moon-111,mars-115,jupiter-119", - hf: defHostFabric, - hs: defHostStorage, + "MS replicas set": { + msReplicas: "moon-111,mars-115,jupiter-119", + hf: defHostFabric, + hs: defHostStorage, expCfg: control.MockServerCfg("ofi+psm2", exmplEngineCfgs). - WithAccessPoints("localhost:10001"). - WithAccessPoints("moon-111:10001", "mars-115:10001", "jupiter-119:10001"). + WithMgmtSvcReplicas("localhost:10001"). + WithMgmtSvcReplicas("moon-111:10001", "mars-115:10001", "jupiter-119:10001"). WithControlLogFile("/tmp/daos_server.log"), }, "unmet min nr ssds": { @@ -391,7 +391,7 @@ func TestDaosServer_Auto_confGen(t *testing.T) { }, }, expCfg: control.MockServerCfg("ofi+psm2", tmpfsEngineCfgs). - WithAccessPoints("localhost:10001"). + WithMgmtSvcReplicas("localhost:10001"). WithControlLogFile("/tmp/daos_server.log"), }, "dcpm scm; control_metadata path set": { @@ -418,7 +418,7 @@ func TestDaosServer_Auto_confGen(t *testing.T) { }, }, expCfg: control.MockServerCfg("ofi+psm2", mdOnSSDEngineCfgs). - WithAccessPoints("localhost:10001"). + WithMgmtSvcReplicas("localhost:10001"). WithControlLogFile("/tmp/daos_server.log"). WithControlMetadata(controlMetadata), }, @@ -510,7 +510,7 @@ func TestDaosServer_Auto_confGen(t *testing.T) { WithBdevDeviceList("0000:97:00.5", "0000:e2:00.5"), ), }). - WithAccessPoints("localhost:10001"). + WithMgmtSvcReplicas("localhost:10001"). WithControlLogFile("/tmp/daos_server.log"), }, } { @@ -522,12 +522,12 @@ func TestDaosServer_Auto_confGen(t *testing.T) { if tc.netClass == "" { tc.netClass = "infiniband" } - if tc.accessPoints == "" { - tc.accessPoints = "localhost" + if tc.msReplicas == "" { + tc.msReplicas = "localhost" } cmd := &configGenCmd{} - cmd.AccessPoints = tc.accessPoints + cmd.MgmtSvcReplicas = tc.msReplicas cmd.NrEngines = tc.nrEngines cmd.SCMOnly = tc.scmOnly cmd.NetClass = tc.netClass @@ -573,7 +573,10 @@ func TestDaosServer_Auto_confGen(t *testing.T) { } return x.Equals(y) }), - cmpopts.IgnoreUnexported(security.CertificateConfig{}), + cmpopts.IgnoreUnexported( + security.CertificateConfig{}, + config.Server{}, + ), } if diff := cmp.Diff(tc.expCfg, gotCfg, cmpOpts...); diff != "" { diff --git a/src/control/cmd/daos_server/config.go b/src/control/cmd/daos_server/config.go index 197e6040458..d58a7abd9ca 100644 --- a/src/control/cmd/daos_server/config.go +++ b/src/control/cmd/daos_server/config.go @@ -11,11 +11,12 @@ import ( "path" "github.com/daos-stack/daos/src/control/build" + "github.com/daos-stack/daos/src/control/logging" "github.com/daos-stack/daos/src/control/server/config" ) type cfgLoader interface { - loadConfig() error + loadConfig(logging.Logger) error configPath() string configOptional() bool } @@ -43,7 +44,7 @@ func (c *cfgCmd) configPath() string { return c.config.Path } -func (c *cfgCmd) loadConfig() error { +func (c *cfgCmd) loadConfig(log logging.Logger) error { if c.IgnoreConfig { c.config = nil return nil @@ -75,7 +76,7 @@ func (c *cfgCmd) loadConfig() error { return err } - return c.config.Load() + return c.config.Load(log) } func (c *cfgCmd) configOptional() bool { diff --git a/src/control/cmd/daos_server/main.go b/src/control/cmd/daos_server/main.go index cc721658711..55d613d9673 100644 --- a/src/control/cmd/daos_server/main.go +++ b/src/control/cmd/daos_server/main.go @@ -168,7 +168,7 @@ func parseOpts(args []string, opts *mainOpts, log *logging.LeveledLogger) error optCfgCmd.setOptional() } - if err := cfgCmd.loadConfig(); err != nil { + if err := cfgCmd.loadConfig(log); err != nil { return errors.Wrapf(err, "failed to load config from %s", cfgCmd.configPath()) } else if cfgCmd.configPath() != "" { diff --git a/src/control/cmd/daos_server/start_test.go b/src/control/cmd/daos_server/start_test.go index 60e79f11387..4c8f419384a 100644 --- a/src/control/cmd/daos_server/start_test.go +++ b/src/control/cmd/daos_server/start_test.go @@ -261,6 +261,7 @@ func TestStartOptions(t *testing.T) { cmpOpts := []cmp.Option{ cmpopts.IgnoreUnexported( security.CertificateConfig{}, + config.Server{}, ), cmpopts.SortSlices(func(a, b string) bool { return a < b }), } diff --git a/src/control/cmd/dmg/auto.go b/src/control/cmd/dmg/auto.go index 3e97a528cc8..d2003ea97ed 100644 --- a/src/control/cmd/dmg/auto.go +++ b/src/control/cmd/dmg/auto.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2020-2023 Intel Corporation. +// (C) Copyright 2020-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -54,6 +54,8 @@ func (cmd *configGenCmd) confGen(ctx context.Context) (*config.Server, error) { hl = []string{"localhost"} } + cmd.CheckDeprecated(cmd.Logger) + req := control.ConfGenerateRemoteReq{ ConfGenerateReq: control.ConfGenerateReq{}, Client: cmd.ctlInvoker, diff --git a/src/control/cmd/dmg/auto_test.go b/src/control/cmd/dmg/auto_test.go index 540c721eff8..800fbed2a65 100644 --- a/src/control/cmd/dmg/auto_test.go +++ b/src/control/cmd/dmg/auto_test.go @@ -77,14 +77,14 @@ func TestAuto_ConfigCommands(t *testing.T) { runConfGenCmdTests(t, []cmdTest{ { - "Generate with no access point", + "Generate with no MS replica", "config generate", printCGRReq(t, func() control.ConfGenerateRemoteReq { req := control.ConfGenerateRemoteReq{ HostList: []string{"localhost:10001"}, } req.ConfGenerateReq.NetClass = hardware.Infiniband - req.ConfGenerateReq.AccessPoints = []string{"localhost"} + req.ConfGenerateReq.MgmtSvcReplicas = []string{"localhost"} return req }()), nil, @@ -100,7 +100,7 @@ func TestAuto_ConfigCommands(t *testing.T) { }, } req.ConfGenerateReq.NetClass = hardware.Infiniband - req.ConfGenerateReq.AccessPoints = []string{"foo"} + req.ConfGenerateReq.MgmtSvcReplicas = []string{"foo"} return req }()), nil, @@ -113,7 +113,7 @@ func TestAuto_ConfigCommands(t *testing.T) { HostList: []string{"localhost:10001"}, } req.ConfGenerateReq.NetClass = hardware.Infiniband - req.ConfGenerateReq.AccessPoints = []string{"foo"} + req.ConfGenerateReq.MgmtSvcReplicas = []string{"foo"} return req }()), nil, @@ -126,7 +126,7 @@ func TestAuto_ConfigCommands(t *testing.T) { HostList: []string{"localhost:10001"}, } req.ConfGenerateReq.NetClass = hardware.Infiniband - req.ConfGenerateReq.AccessPoints = []string{"foo"} + req.ConfGenerateReq.MgmtSvcReplicas = []string{"foo"} req.ConfGenerateReq.SCMOnly = true return req }()), @@ -140,7 +140,7 @@ func TestAuto_ConfigCommands(t *testing.T) { HostList: []string{"localhost:10001"}, } req.ConfGenerateReq.NetClass = hardware.Infiniband - req.ConfGenerateReq.AccessPoints = []string{"foo"} + req.ConfGenerateReq.MgmtSvcReplicas = []string{"foo"} req.ConfGenerateReq.NrEngines = 2 return req }()), @@ -154,7 +154,7 @@ func TestAuto_ConfigCommands(t *testing.T) { HostList: []string{"localhost:10001"}, } req.ConfGenerateReq.NetClass = hardware.Infiniband - req.ConfGenerateReq.AccessPoints = []string{"foo"} + req.ConfGenerateReq.MgmtSvcReplicas = []string{"foo"} req.ConfGenerateReq.NrEngines = 2 req.ConfGenerateReq.SCMOnly = true return req @@ -169,7 +169,7 @@ func TestAuto_ConfigCommands(t *testing.T) { HostList: []string{"localhost:10001"}, } req.ConfGenerateReq.NetClass = hardware.Ether - req.ConfGenerateReq.AccessPoints = []string{"foo"} + req.ConfGenerateReq.MgmtSvcReplicas = []string{"foo"} return req }()), nil, @@ -182,7 +182,7 @@ func TestAuto_ConfigCommands(t *testing.T) { HostList: []string{"localhost:10001"}, } req.ConfGenerateReq.NetClass = hardware.Infiniband - req.ConfGenerateReq.AccessPoints = []string{"foo"} + req.ConfGenerateReq.MgmtSvcReplicas = []string{"foo"} return req }()), nil, @@ -207,7 +207,7 @@ func TestAuto_ConfigCommands(t *testing.T) { HostList: []string{"localhost:10001"}, } req.ConfGenerateReq.NetClass = hardware.Infiniband - req.ConfGenerateReq.AccessPoints = []string{"foo"} + req.ConfGenerateReq.MgmtSvcReplicas = []string{"foo"} req.ConfGenerateReq.FabricPorts = []int{12345, 13345} return req }()), @@ -221,7 +221,7 @@ func TestAuto_ConfigCommands(t *testing.T) { HostList: []string{"localhost:10001"}, } req.ConfGenerateReq.NetClass = hardware.Infiniband - req.ConfGenerateReq.AccessPoints = []string{"foo"} + req.ConfGenerateReq.MgmtSvcReplicas = []string{"foo"} req.ConfGenerateReq.UseTmpfsSCM = true return req }()), @@ -235,7 +235,7 @@ func TestAuto_ConfigCommands(t *testing.T) { HostList: []string{"localhost:10001"}, } req.ConfGenerateReq.NetClass = hardware.Infiniband - req.ConfGenerateReq.AccessPoints = []string{"foo"} + req.ConfGenerateReq.MgmtSvcReplicas = []string{"foo"} req.ConfGenerateReq.UseTmpfsSCM = true req.ConfGenerateReq.ExtMetadataPath = "/opt/daos" return req @@ -256,7 +256,7 @@ func TestAuto_confGenCmd_Convert(t *testing.T) { cmd.NrEngines = 1 cmd.NetProvider = "ofi+tcp" cmd.SCMOnly = true - cmd.AccessPoints = "foo,bar" + cmd.MgmtSvcReplicas = "foo,bar" cmd.NetClass = "infiniband" cmd.UseTmpfsSCM = true cmd.ExtMetadataPath = "/opt/daos_md" @@ -271,7 +271,7 @@ func TestAuto_confGenCmd_Convert(t *testing.T) { NrEngines: 1, NetProvider: "ofi+tcp", SCMOnly: true, - AccessPoints: []string{"foo", "bar"}, + MgmtSvcReplicas: []string{"foo", "bar"}, NetClass: hardware.Infiniband, UseTmpfsSCM: true, ExtMetadataPath: "/opt/daos_md", @@ -338,7 +338,7 @@ func TestAuto_confGen(t *testing.T) { for name, tc := range map[string]struct { hostlist []string - accessPoints string + msReplicas string nrEngines int scmOnly bool netClass string @@ -376,17 +376,17 @@ func TestAuto_confGen(t *testing.T) { {storHostResp}, }, expCfg: control.MockServerCfg("ofi+psm2", exmplEngineCfgs). - WithAccessPoints("localhost:10001"). + WithMgmtSvcReplicas("localhost:10001"). WithControlLogFile("/tmp/daos_server.log"), }, - "dcpm scm; access points set": { - accessPoints: "moon-111,mars-115,jupiter-119", + "dcpm scm; MS replicas set": { + msReplicas: "moon-111,mars-115,jupiter-119", hostResponsesSet: [][]*control.HostResponse{ {netHostResp}, {storHostResp}, }, expCfg: control.MockServerCfg("ofi+psm2", exmplEngineCfgs). - WithAccessPoints("moon-111:10001", "mars-115:10001", "jupiter-119:10001"). + WithMgmtSvcReplicas("moon-111:10001", "mars-115:10001", "jupiter-119:10001"). WithControlLogFile("/tmp/daos_server.log"), }, "dcpm scm; unmet min nr ssds": { @@ -466,11 +466,11 @@ func TestAuto_confGen(t *testing.T) { if tc.netClass == "" { tc.netClass = "infiniband" } - if tc.accessPoints == "" { - tc.accessPoints = "localhost" + if tc.msReplicas == "" { + tc.msReplicas = "localhost" } cmd := &configGenCmd{} - cmd.AccessPoints = tc.accessPoints + cmd.MgmtSvcReplicas = tc.msReplicas cmd.NrEngines = tc.nrEngines cmd.SCMOnly = tc.scmOnly cmd.NetClass = tc.netClass @@ -520,7 +520,10 @@ func TestAuto_confGen(t *testing.T) { } return x.Equals(y) }), - cmpopts.IgnoreUnexported(security.CertificateConfig{}), + cmpopts.IgnoreUnexported( + security.CertificateConfig{}, + config.Server{}, + ), } if diff := cmp.Diff(tc.expCfg, gotCfg, cmpOpts...); diff != "" { @@ -596,7 +599,7 @@ core_dump_filter: 19 name: daos_server socket_dir: /var/run/daos_server provider: ofi+verbs -access_points: +mgmt_svc_replicas: - hostX:10002 fault_cb: "" hyperthreads: false @@ -606,7 +609,7 @@ hyperthreads: false typicalAutoGenOutCfg := config.DefaultServer(). WithControlLogFile(defaultControlLogFile). WithFabricProvider("ofi+verbs"). - WithAccessPoints("hostX:10002"). + WithMgmtSvcReplicas("hostX:10002"). WithDisableVMD(false). WithEngines( engine.MockConfig(). diff --git a/src/control/common/cmdutil/auto.go b/src/control/common/cmdutil/auto.go index 1081942e2c0..26bd56eba49 100644 --- a/src/control/common/cmdutil/auto.go +++ b/src/control/common/cmdutil/auto.go @@ -1,7 +1,14 @@ package cmdutil +import "github.com/daos-stack/daos/src/control/logging" + +type deprecatedParams struct { + AccessPoints string `short:"a" long:"access-points" description:"DEPRECATED; use ms-replicas instead" json:",omitempty"` // deprecated in 2.8 +} + type ConfGenCmd struct { - AccessPoints string `default:"localhost" short:"a" long:"access-points" description:"Comma separated list of access point addresses to host management service"` + deprecatedParams + MgmtSvcReplicas string `default:"localhost" short:"r" long:"ms-replicas" description:"Comma separated list of MS replica addresses to host management service"` NrEngines int `short:"e" long:"num-engines" description:"Set the number of DAOS Engine sections to be populated in the config file output. If unset then the value will be set to the number of NUMA nodes on storage hosts in the DAOS system."` SCMOnly bool `short:"s" long:"scm-only" description:"Create a SCM-only config without NVMe SSDs."` NetClass string `default:"infiniband" short:"c" long:"net-class" description:"Set the network device class to be used" choice:"ethernet" choice:"infiniband"` @@ -10,3 +17,12 @@ type ConfGenCmd struct { ExtMetadataPath string `short:"m" long:"control-metadata-path" description:"External storage path to store control metadata. Set this to a persistent location and specify --use-tmpfs-scm to create an MD-on-SSD config"` FabricPorts string `short:"f" long:"fabric-ports" description:"Allow custom fabric interface ports to be specified for each engine config section. Comma separated port numbers, one per engine"` } + +// CheckDeprecated will check for deprecated parameters and update as needed. +func (cmd *ConfGenCmd) CheckDeprecated(log logging.Logger) { + if cmd.AccessPoints != "" { + log.Notice("access-points is deprecated; please use ms-replicas instead") + cmd.MgmtSvcReplicas = cmd.AccessPoints + cmd.AccessPoints = "" + } +} diff --git a/src/control/fault/code/codes.go b/src/control/fault/code/codes.go index 4d045f7cfce..bbbb5d76cdf 100644 --- a/src/control/fault/code/codes.go +++ b/src/control/fault/code/codes.go @@ -165,8 +165,8 @@ const ( ServerNoConfigPath ServerConfigBadControlPort ServerConfigBadTelemetryPort - ServerConfigBadAccessPoints - ServerConfigEvenAccessPoints + ServerConfigBadMgmtSvcReplicas + ServerConfigEvenMgmtSvcReplicas ServerConfigBadProvider ServerConfigNoEngines ServerConfigDuplicateFabric diff --git a/src/control/lib/control/auto.go b/src/control/lib/control/auto.go index 0f2a94c404c..a5e981cb6a9 100644 --- a/src/control/lib/control/auto.go +++ b/src/control/lib/control/auto.go @@ -63,7 +63,7 @@ type ( // Generate a config without NVMe. SCMOnly bool `json:"SCMOnly"` // Hosts to run the management service. - AccessPoints []string `json:"-"` + MgmtSvcReplicas []string `json:"-"` // Ports to use for fabric comms (one needed per engine). FabricPorts []int `json:"-"` // Generate config with a tmpfs RAM-disk SCM. @@ -101,9 +101,9 @@ type ( func (cgr *ConfGenerateReq) UnmarshalJSON(data []byte) error { type Alias ConfGenerateReq aux := &struct { - AccessPoints string - FabricPorts string - NetClass string + MgmtSvcReplicas string + FabricPorts string + NetClass string *Alias }{ Alias: (*Alias)(cgr), @@ -113,7 +113,7 @@ func (cgr *ConfGenerateReq) UnmarshalJSON(data []byte) error { return err } - cgr.AccessPoints = strings.Split(aux.AccessPoints, ",") + cgr.MgmtSvcReplicas = strings.Split(aux.MgmtSvcReplicas, ",") fabricPorts := strings.Split(aux.FabricPorts, ",") for _, s := range fabricPorts { if s == "" { @@ -212,8 +212,8 @@ func ConfGenerateRemote(ctx context.Context, req ConfGenerateRemoteReq) (*ConfGe return nil, errors.New("no hosts specified") } - if len(req.AccessPoints) == 0 { - return nil, errors.New("no access points specified") + if len(req.MgmtSvcReplicas) == 0 { + return nil, errors.New("no MS replicas specified") } ns, err := getNetworkSet(ctx, req) @@ -1190,24 +1190,24 @@ func getThreadCounts(log logging.Logger, nodeSet []int, coresPerEngine int, numa return &tc, nil } -// check that all access points either have no port specified or have the same port number. -func checkAccessPointPorts(log logging.Logger, aps []string) (int, error) { - if len(aps) == 0 { - return 0, errors.New("no access points") +// check that all MS replicas either have no port specified or have the same port number. +func checkReplicaPorts(log logging.Logger, replicas []string) (int, error) { + if len(replicas) == 0 { + return 0, errors.New("no MS replicas") } port := -1 - for _, ap := range aps { - apPort, err := config.GetAccessPointPort(log, ap) + for _, ap := range replicas { + apPort, err := config.GetMSReplicaPort(log, ap) if err != nil { - return 0, errors.Wrapf(err, "access point %q", ap) + return 0, errors.Wrapf(err, "MS replica %q", ap) } if port == -1 { port = apPort continue } if apPort != port { - return 0, errors.New("access point port numbers do not match") + return 0, errors.New("MS replica port numbers do not match") } } @@ -1228,7 +1228,7 @@ func genServerConfig(req ConfGenerateReq, ecs []*engine.Config, tc *threadCounts } cfg := config.DefaultServer(). - WithAccessPoints(req.AccessPoints...). + WithMgmtSvcReplicas(req.MgmtSvcReplicas...). WithFabricProvider(ecs[0].Fabric.Provider). WithEngines(ecs...). WithControlLogFile(defaultControlLogFile) @@ -1247,12 +1247,12 @@ func genServerConfig(req ConfGenerateReq, ecs []*engine.Config, tc *threadCounts } } - portNum, err := checkAccessPointPorts(req.Log, cfg.AccessPoints) + portNum, err := checkReplicaPorts(req.Log, cfg.MgmtSvcReplicas) if err != nil { return nil, err } if portNum != 0 { - // Custom access point port number specified so set server port to the same. + // Custom MS replica port number specified so set server port to the same. cfg.WithControlPort(portNum) } diff --git a/src/control/lib/control/auto_test.go b/src/control/lib/control/auto_test.go index 724bed72180..41dd4a248c3 100644 --- a/src/control/lib/control/auto_test.go +++ b/src/control/lib/control/auto_test.go @@ -1621,7 +1621,7 @@ func TestControl_AutoConfig_genServerConfig(t *testing.T) { } for name, tc := range map[string]struct { - accessPoints []string // list of access point host/ip addresses + msReplicas []string // list of MS replica host/ip addresses extMetadataPath string ecs []*engine.Config threadCounts *threadCounts // numa to cpu mappings @@ -1638,46 +1638,46 @@ func TestControl_AutoConfig_genServerConfig(t *testing.T) { }, expErr: errors.New("provider not specified"), }, - "no access points": { - accessPoints: []string{}, + "no MS replicas": { + msReplicas: []string{}, threadCounts: &threadCounts{16, 0}, ecs: []*engine.Config{exmplEngineCfg0}, - expErr: errors.New("no access points"), + expErr: errors.New("no MS replicas"), }, - "access points without the same port": { - accessPoints: []string{"bob:1", "joe:2"}, + "MS replicas without the same port": { + msReplicas: []string{"bob:1", "joe:2"}, threadCounts: &threadCounts{16, 0}, ecs: []*engine.Config{exmplEngineCfg0}, expErr: errors.New("numbers do not match"), }, - "access points some with port specified": { - accessPoints: []string{"bob:1", "joe"}, + "MS replicas some with port specified": { + msReplicas: []string{"bob:1", "joe"}, threadCounts: &threadCounts{16, 0}, ecs: []*engine.Config{exmplEngineCfg0}, expErr: errors.New("numbers do not match"), }, "single engine config; default port number": { - accessPoints: []string{"hostX"}, + msReplicas: []string{"hostX"}, threadCounts: &threadCounts{16, 0}, ecs: []*engine.Config{exmplEngineCfg0}, expCfg: MockServerCfg(exmplEngineCfg0.Fabric.Provider, []*engine.Config{ exmplEngineCfg0.WithHelperStreamCount(0), }). - WithAccessPoints("hostX:10001"), // Default applied. + WithMgmtSvcReplicas("hostX:10001"), // Default applied. }, "single engine config; default port number specified": { - accessPoints: []string{"hostX:10001"}, + msReplicas: []string{"hostX:10001"}, threadCounts: &threadCounts{16, 0}, ecs: []*engine.Config{exmplEngineCfg0}, expCfg: MockServerCfg(exmplEngineCfg0.Fabric.Provider, []*engine.Config{ exmplEngineCfg0.WithHelperStreamCount(0), }). - WithAccessPoints("hostX:10001"), // ControlPort remains at 10001. + WithMgmtSvcReplicas("hostX:10001"), // ControlPort remains at 10001. }, - "dual engine config; custom access point port number": { - accessPoints: []string{"hostX:10002"}, + "dual engine config; custom MS replica port number": { + msReplicas: []string{"hostX:10002"}, threadCounts: &threadCounts{16, 0}, ecs: []*engine.Config{ exmplEngineCfg0, @@ -1688,11 +1688,11 @@ func TestControl_AutoConfig_genServerConfig(t *testing.T) { exmplEngineCfg0.WithHelperStreamCount(0), exmplEngineCfg1.WithHelperStreamCount(0), }). - WithAccessPoints("hostX:10002"). + WithMgmtSvcReplicas("hostX:10002"). WithControlPort(10002), // ControlPort updated to AP port. }, - "bad accesspoint port": { - accessPoints: []string{"hostX:-10001"}, + "bad MS replica port": { + msReplicas: []string{"hostX:-10001"}, threadCounts: &threadCounts{16, 0}, ecs: []*engine.Config{ exmplEngineCfg0, @@ -1709,7 +1709,7 @@ func TestControl_AutoConfig_genServerConfig(t *testing.T) { expErr: errors.New("multiple bdev tiers"), }, "dual engine tmpfs; high mem": { - accessPoints: []string{"hostX:10002", "hostY:10002", "hostZ:10002"}, + msReplicas: []string{"hostX:10002", "hostY:10002", "hostZ:10002"}, extMetadataPath: metadataMountPath, threadCounts: &threadCounts{16, 0}, ecs: []*engine.Config{ @@ -1737,7 +1737,7 @@ func TestControl_AutoConfig_genServerConfig(t *testing.T) { storage.BdevOutConfName), ), }). - WithAccessPoints("hostX:10002", "hostY:10002", "hostZ:10002"). + WithMgmtSvcReplicas("hostX:10002", "hostY:10002", "hostZ:10002"). WithControlPort(10002). // ControlPort updated to AP port. WithControlMetadata(controlMetadata), }, @@ -1746,8 +1746,8 @@ func TestControl_AutoConfig_genServerConfig(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) defer test.ShowBufferOnFailure(t, buf) - if tc.accessPoints == nil { - tc.accessPoints = []string{"localhost"} // Matches default in mock config. + if tc.msReplicas == nil { + tc.msReplicas = []string{"localhost"} // Matches default in mock config. } if tc.threadCounts == nil { tc.threadCounts = &threadCounts{} @@ -1755,7 +1755,7 @@ func TestControl_AutoConfig_genServerConfig(t *testing.T) { req := ConfGenerateReq{ Log: log, - AccessPoints: tc.accessPoints, + MgmtSvcReplicas: tc.msReplicas, ExtMetadataPath: tc.extMetadataPath, } @@ -1772,7 +1772,10 @@ func TestControl_AutoConfig_genServerConfig(t *testing.T) { } return x.Equals(y) }), - cmpopts.IgnoreUnexported(security.CertificateConfig{}), + cmpopts.IgnoreUnexported( + security.CertificateConfig{}, + config.Server{}, + ), } cmpOpts = append(cmpOpts, defResCmpOpts()...) diff --git a/src/control/lib/control/event.go b/src/control/lib/control/event.go index 2a9896873c3..d316a4e7cb1 100644 --- a/src/control/lib/control/event.go +++ b/src/control/lib/control/event.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2021-2022 Intel Corporation. +// (C) Copyright 2021-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -68,11 +68,11 @@ func eventNotify(ctx context.Context, rpcClient UnaryInvoker, seq uint64, evt *e } // EventForwarder implements the events.Handler interface, increments sequence -// number for each event forwarded and distributes requests to MS access points. +// number for each event forwarded and distributes requests to MS replicas. type EventForwarder struct { - seq <-chan uint64 - client UnaryInvoker - accessPts []string + seq <-chan uint64 + client UnaryInvoker + msReplicas []string } // OnEvent implements the events.Handler interface. @@ -81,21 +81,21 @@ func (ef *EventForwarder) OnEvent(ctx context.Context, evt *events.RASEvent) { case evt == nil: ef.client.Debug("skip event forwarding, nil event") return - case len(ef.accessPts) == 0: - ef.client.Debug("skip event forwarding, missing access points") + case len(ef.msReplicas) == 0: + ef.client.Debug("skip event forwarding, missing MS replicas") return case !evt.ShouldForward(): ef.client.Debugf("forwarding disabled for %s event", evt.ID) return } - if err := eventNotify(ctx, ef.client, <-ef.seq, evt, ef.accessPts); err != nil { + if err := eventNotify(ctx, ef.client, <-ef.seq, evt, ef.msReplicas); err != nil { ef.client.Debugf("failed to forward event to MS: %s", err) } } // NewEventForwarder returns an initialized EventForwarder. -func NewEventForwarder(rpcClient UnaryInvoker, accessPts []string) *EventForwarder { +func NewEventForwarder(rpcClient UnaryInvoker, replicas []string) *EventForwarder { seqCh := make(chan uint64) go func(ch chan<- uint64) { for i := uint64(1); ; i++ { @@ -104,9 +104,9 @@ func NewEventForwarder(rpcClient UnaryInvoker, accessPts []string) *EventForward }(seqCh) return &EventForwarder{ - seq: seqCh, - client: rpcClient, - accessPts: accessPts, + seq: seqCh, + client: rpcClient, + msReplicas: replicas, } } diff --git a/src/control/lib/control/event_test.go b/src/control/lib/control/event_test.go index c8cd47d6fee..73b60ae1873 100644 --- a/src/control/lib/control/event_test.go +++ b/src/control/lib/control/event_test.go @@ -1,5 +1,5 @@ // -// (C) Copyright 2021-2022 Intel Corporation. +// (C) Copyright 2021-2024 Intel Corporation. // // SPDX-License-Identifier: BSD-2-Clause-Patent // @@ -89,7 +89,7 @@ func TestControl_EventForwarder_OnEvent(t *testing.T) { rasEventEngineDiedFwdable := mockEvtEngineDied(t).WithForwardable(true) for name, tc := range map[string]struct { - aps []string + replicas []string event *events.RASEvent nilClient bool expInvokeCount int @@ -97,17 +97,17 @@ func TestControl_EventForwarder_OnEvent(t *testing.T) { "nil event": { event: nil, }, - "missing access points": { + "missing MS replicas": { event: rasEventEngineDiedFwdable, }, "successful forward": { event: rasEventEngineDiedFwdable, - aps: []string{"192.168.1.1"}, + replicas: []string{"192.168.1.1"}, expInvokeCount: 2, }, "skip non-forwardable event": { - event: rasEventEngineDied, - aps: []string{"192.168.1.1"}, + event: rasEventEngineDied, + replicas: []string{"192.168.1.1"}, }, } { t.Run(name, func(t *testing.T) { @@ -126,7 +126,7 @@ func TestControl_EventForwarder_OnEvent(t *testing.T) { callCount++ // call at least once } - ef := NewEventForwarder(mi, tc.aps) + ef := NewEventForwarder(mi, tc.replicas) for i := 0; i < callCount; i++ { ef.OnEvent(test.Context(t), tc.event) } diff --git a/src/control/lib/support/log.go b/src/control/lib/support/log.go index 02bf0abdb54..f6446159d75 100644 --- a/src/control/lib/support/log.go +++ b/src/control/lib/support/log.go @@ -464,7 +464,7 @@ func rsyncLog(log logging.Logger, opts ...CollectLogsParams) error { if cfgPath != "" { serverConfig := config.DefaultServer() serverConfig.SetPath(cfgPath) - if err := serverConfig.Load(); err == nil { + if err := serverConfig.Load(log); err == nil { if serverConfig.SupportConfig.FileTransferExec != "" { return customCopy(log, opts[0], serverConfig.SupportConfig.FileTransferExec) } @@ -682,7 +682,7 @@ func copyServerConfig(log logging.Logger, opts ...CollectLogsParams) error { serverConfig := config.DefaultServer() serverConfig.SetPath(cfgPath) - serverConfig.Load() + serverConfig.Load(log) // Create the individual folder on each server targetConfig, err := createHostLogFolder(DaosServerConfig, log, opts...) if err != nil { @@ -862,7 +862,7 @@ func collectServerLog(log logging.Logger, opts ...CollectLogsParams) error { } serverConfig := config.DefaultServer() serverConfig.SetPath(cfgPath) - serverConfig.Load() + serverConfig.Load(log) switch opts[0].LogCmd { case "EngineLog": @@ -928,7 +928,7 @@ func collectDaosMetrics(daosNodeLocation string, log logging.Logger, opts ...Col } serverConfig := config.DefaultServer() serverConfig.SetPath(cfgPath) - serverConfig.Load() + serverConfig.Load(log) for i := range serverConfig.Engines { engineId := fmt.Sprintf("%d", i) diff --git a/src/control/server/config/faults.go b/src/control/server/config/faults.go index b5128ffcb20..c2f2063357a 100644 --- a/src/control/server/config/faults.go +++ b/src/control/server/config/faults.go @@ -41,15 +41,15 @@ var ( "invalid telemetry port in configuration", "specify a positive non-zero network port in configuration ('telemetry_port' parameter) and restart the control server", ) - FaultConfigBadAccessPoints = serverConfigFault( - code.ServerConfigBadAccessPoints, - "invalid list of access points in configuration", - "'access_points' must contain resolvable addresses; fix the configuration and restart the control server", - ) - FaultConfigEvenAccessPoints = serverConfigFault( - code.ServerConfigEvenAccessPoints, - "non-odd number of access points in configuration", - "'access_points' must contain an odd number (e.g. 1, 3, 5, etc.) of addresses; fix the configuration and restart the control server", + FaultConfigBadMgmtSvcReplicas = serverConfigFault( + code.ServerConfigBadMgmtSvcReplicas, + "invalid list of MS replicas in configuration", + "'mgmt_svc_replicas' must contain resolvable addresses; fix the configuration and restart the control server", + ) + FaultConfigEvenMgmtSvcReplicas = serverConfigFault( + code.ServerConfigEvenMgmtSvcReplicas, + "non-odd number of MS replicas in configuration", + "'mgmt_svc_replicas' must contain an odd number (e.g. 1, 3, 5, etc.) of addresses; fix the configuration and restart the control server", ) FaultConfigNoProvider = serverConfigFault( code.ServerConfigBadProvider, diff --git a/src/control/server/config/server.go b/src/control/server/config/server.go index ec94784d7d5..4b4c5bd529c 100644 --- a/src/control/server/config/server.go +++ b/src/control/server/config/server.go @@ -41,6 +41,10 @@ type SupportConfig struct { FileTransferExec string `yaml:"file_transfer_exec,omitempty"` } +type deprecatedParams struct { + AccessPoints []string `yaml:"access_points,omitempty"` // deprecated in 2.8 +} + // Server describes configuration options for DAOS control plane. // See utils/config/daos_server.yml for parameter descriptions. type Server struct { @@ -72,7 +76,7 @@ type Server struct { Fabric engine.FabricConfig `yaml:",inline"` Modules string `yaml:"-"` - AccessPoints []string `yaml:"access_points"` + MgmtSvcReplicas []string `yaml:"mgmt_svc_replicas"` Metadata storage.ControlMetadata `yaml:"control_metadata,omitempty"` @@ -84,6 +88,8 @@ type Server struct { // Behavior flags AutoFormat bool `yaml:"-"` + + deprecatedParams `yaml:",inline"` } // WithCoreDumpFilter sets the core dump filter written to /proc/self/coredump_filter. @@ -200,9 +206,9 @@ func (cfg *Server) WithEngines(engineList ...*engine.Config) *Server { return cfg } -// WithAccessPoints sets the access point list. -func (cfg *Server) WithAccessPoints(aps ...string) *Server { - cfg.AccessPoints = aps +// WithMgmtSvcReplicas sets the MS replicas list. +func (cfg *Server) WithMgmtSvcReplicas(reps ...string) *Server { + cfg.MgmtSvcReplicas = reps return cfg } @@ -324,7 +330,7 @@ func DefaultServer() *Server { return &Server{ SystemName: build.DefaultSystemName, SocketDir: defaultRuntimeDir, - AccessPoints: []string{fmt.Sprintf("localhost:%d", build.DefaultControlPort)}, + MgmtSvcReplicas: []string{fmt.Sprintf("localhost:%d", build.DefaultControlPort)}, ControlPort: build.DefaultControlPort, TransportConfig: security.DefaultServerTransportConfig(), Hyperthreads: false, @@ -338,7 +344,7 @@ func DefaultServer() *Server { } // Load reads the serialized configuration from disk and validates file syntax. -func (cfg *Server) Load() error { +func (cfg *Server) Load(log logging.Logger) error { if cfg.Path == "" { return FaultConfigNoPath } @@ -372,6 +378,12 @@ func (cfg *Server) Load() error { cfg.ClientEnvVars = common.MergeKeyValues(cfg.ClientEnvVars, []string{cfg.Fabric.GetAuthKeyEnv()}) } + if len(cfg.deprecatedParams.AccessPoints) > 0 { + log.Notice("access_points is deprecated; please use mgmt_svc_replicas instead") + cfg.MgmtSvcReplicas = cfg.deprecatedParams.AccessPoints + cfg.deprecatedParams.AccessPoints = nil + } + return nil } @@ -413,22 +425,22 @@ func (cfg *Server) SaveActiveConfig(log logging.Logger) { log.Debugf("active config saved to %s (read-only)", activeConfig) } -// GetAccessPointPort returns port number suffixed to AP address after its validation or 0 if no +// GetMSReplicaPort returns port number suffixed to replicas address after its validation or 0 if no // port number specified. Error returned if validation fails. -func GetAccessPointPort(log logging.Logger, addr string) (int, error) { +func GetMSReplicaPort(log logging.Logger, addr string) (int, error) { if !common.HasPort(addr) { return 0, nil } _, port, err := net.SplitHostPort(addr) if err != nil { - log.Errorf("invalid access point %q: %s", addr, err) - return 0, FaultConfigBadAccessPoints + log.Errorf("invalid MS replica %q: %s", addr, err) + return 0, FaultConfigBadMgmtSvcReplicas } portNum, err := strconv.Atoi(port) if err != nil { - log.Errorf("invalid access point port: %s", err) + log.Errorf("invalid MS replica port: %s", err) return 0, FaultConfigBadControlPort } if portNum <= 0 { @@ -436,17 +448,17 @@ func GetAccessPointPort(log logging.Logger, addr string) (int, error) { if portNum < 0 { m = "negative" } - log.Errorf("access point port cannot be %s", m) + log.Errorf("MS replica port cannot be %s", m) return 0, FaultConfigBadControlPort } return portNum, nil } -// getAccessPointAddrWithPort appends default port number to address if custom port is not +// getReplicaAddrWithPort appends default port number to address if custom port is not // specified, otherwise custom specified port is validated. -func getAccessPointAddrWithPort(log logging.Logger, addr string, portDefault int) (string, error) { - portNum, err := GetAccessPointPort(log, addr) +func getReplicaAddrWithPort(log logging.Logger, addr string, portDefault int) (string, error) { + portNum, err := GetMSReplicaPort(log, addr) if err != nil { return "", err } @@ -454,9 +466,9 @@ func getAccessPointAddrWithPort(log logging.Logger, addr string, portDefault int return fmt.Sprintf("%s:%d", addr, portDefault), nil } - // Warn if access point port differs from config control port. + // Warn if MS replica port differs from config control port. if portDefault != portNum { - log.Debugf("access point %q port differs from default port %q", + log.Debugf("ms replica %q port differs from default port %q", addr, portDefault) } @@ -653,20 +665,20 @@ func (cfg *Server) Validate(log logging.Logger) (err error) { log.Debugf("vfio=%v hotplug=%v vmd=%v requested in config", !cfg.DisableVFIO, cfg.EnableHotplug, !(*cfg.DisableVMD)) - // Update access point addresses with control port if port is not supplied. - newAPs := make([]string, 0, len(cfg.AccessPoints)) - for _, ap := range cfg.AccessPoints { - newAP, err := getAccessPointAddrWithPort(log, ap, cfg.ControlPort) + // Update MS replica addresses with control port if port is not supplied. + newReps := make([]string, 0, len(cfg.MgmtSvcReplicas)) + for _, rep := range cfg.MgmtSvcReplicas { + newAP, err := getReplicaAddrWithPort(log, rep, cfg.ControlPort) if err != nil { return err } - newAPs = append(newAPs, newAP) + newReps = append(newReps, newAP) } - if common.StringSliceHasDuplicates(newAPs) { - log.Error("duplicate access points addresses") - return FaultConfigBadAccessPoints + if common.StringSliceHasDuplicates(newReps) { + log.Error("duplicate MS replica addresses") + return FaultConfigBadMgmtSvcReplicas } - cfg.AccessPoints = newAPs + cfg.MgmtSvcReplicas = newReps if cfg.Metadata.DevicePath != "" && cfg.Metadata.Path == "" { return FaultConfigControlMetadataNoPath @@ -686,13 +698,13 @@ func (cfg *Server) Validate(log logging.Logger) (err error) { } switch { - case len(cfg.AccessPoints) < 1: - return FaultConfigBadAccessPoints - case len(cfg.AccessPoints)%2 == 0: - return FaultConfigEvenAccessPoints - case len(cfg.AccessPoints) == 1: - log.Noticef("Configuration includes only one access point. This provides no redundancy " + - "in the event of an access point failure.") + case len(cfg.MgmtSvcReplicas) < 1: + return FaultConfigBadMgmtSvcReplicas + case len(cfg.MgmtSvcReplicas)%2 == 0: + return FaultConfigEvenMgmtSvcReplicas + case len(cfg.MgmtSvcReplicas) == 1: + log.Noticef("Configuration includes only one MS replica. This provides no redundancy " + + "in the event of a MS replica failure.") } switch { diff --git a/src/control/server/config/server_test.go b/src/control/server/config/server_test.go index 6961a0f4190..034d21a7b5d 100644 --- a/src/control/server/config/server_test.go +++ b/src/control/server/config/server_test.go @@ -44,6 +44,7 @@ var ( cmpopts.SortSlices(func(x, y string) bool { return x < y }), cmpopts.IgnoreUnexported( security.CertificateConfig{}, + Server{}, ), cmpopts.IgnoreFields(Server{}, "Path"), cmp.Comparer(func(x, y *storage.BdevDeviceList) bool { @@ -55,10 +56,10 @@ var ( } ) -func baseCfg(t *testing.T, testFile string) *Server { +func baseCfg(t *testing.T, log logging.Logger, testFile string) *Server { t.Helper() - config, err := mockConfigFromFile(t, testFile) + config, err := mockConfigFromFile(t, log, testFile) if err != nil { t.Fatalf("failed to load %s: %s", testFile, err) } @@ -125,12 +126,12 @@ func uncommentServerConfig(t *testing.T, outFile string) { // mockConfigFromFile returns a populated server config file from the // file at the given path. -func mockConfigFromFile(t *testing.T, path string) (*Server, error) { +func mockConfigFromFile(t *testing.T, log logging.Logger, path string) (*Server, error) { t.Helper() c := DefaultServer() c.Path = path - return c, c.Load() + return c, c.Load(log) } func TestServerConfig_MarshalUnmarshal(t *testing.T) { @@ -163,7 +164,7 @@ func TestServerConfig_MarshalUnmarshal(t *testing.T) { configA := DefaultServer() configA.Path = tt.inPath - err := configA.Load() + err := configA.Load(log) if err == nil { err = configA.Validate(log) } @@ -194,7 +195,7 @@ func TestServerConfig_MarshalUnmarshal(t *testing.T) { t.Fatal(err) } - err = configB.Load() + err = configB.Load(log) if err == nil { err = configB.Validate(log) } @@ -220,10 +221,13 @@ func TestServerConfig_Constructed(t *testing.T) { testDir, cleanup := test.CreateTestDir(t) defer cleanup() + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + // First, load a config based on the server config with all options uncommented. testFile := filepath.Join(testDir, sConfigUncomment) uncommentServerConfig(t, testFile) - defaultCfg, err := mockConfigFromFile(t, testFile) + defaultCfg, err := mockConfigFromFile(t, log, testFile) if err != nil { t.Fatalf("failed to load %s: %s", testFile, err) } @@ -251,7 +255,7 @@ func TestServerConfig_Constructed(t *testing.T) { WithSocketDir("./.daos/daos_server"). WithFabricProvider("ofi+verbs;ofi_rxm"). WithCrtTimeout(30). - WithAccessPoints("hostname1"). + WithMgmtSvcReplicas("hostname1", "hostname2", "hostname3"). WithFaultCb("./.daos/fd_callback"). WithFaultPath("/vcdu0/rack1/hostname"). WithClientEnvVars([]string{"foo=bar"}). @@ -407,7 +411,7 @@ func TestServerConfig_MDonSSD_Constructed(t *testing.T) { log, buf := logging.NewTestLogger(t.Name()) defer test.ShowBufferOnFailure(t, buf) - mdOnSSDCfg, err := mockConfigFromFile(t, mdOnSSDExample) + mdOnSSDCfg, err := mockConfigFromFile(t, log, mdOnSSDExample) if err != nil { t.Fatalf("failed to load %s: %s", mdOnSSDExample, err) } @@ -419,7 +423,7 @@ func TestServerConfig_MDonSSD_Constructed(t *testing.T) { WithControlLogFile("/tmp/daos_server.log"). WithTelemetryPort(9191). WithFabricProvider("ofi+tcp"). - WithAccessPoints("example") + WithMgmtSvcReplicas("example1", "example2", "example3") constructed.Engines = []*engine.Config{ engine.MockConfig(). @@ -471,6 +475,9 @@ func TestServerConfig_Validation(t *testing.T) { testDir, cleanup := test.CreateTestDir(t) defer cleanup() + log, buf := logging.NewTestLogger(t.Name()) + defer test.ShowBufferOnFailure(t, buf) + // First, load a config based on the server config with all options uncommented. testFile := filepath.Join(testDir, sConfigUncomment) uncommentServerConfig(t, testFile) @@ -503,90 +510,90 @@ func TestServerConfig_Validation(t *testing.T) { }, expErr: FaultConfigNoProvider, }, - "no access point": { + "no MS replica": { extraConfig: func(c *Server) *Server { - return c.WithAccessPoints() + return c.WithMgmtSvcReplicas() }, - expErr: FaultConfigBadAccessPoints, + expErr: FaultConfigBadMgmtSvcReplicas, }, - "single access point": { + "single MS replica": { extraConfig: func(c *Server) *Server { - return c.WithAccessPoints("1.2.3.4:1234") + return c.WithMgmtSvcReplicas("1.2.3.4:1234") }, }, - "multiple access points (even)": { + "multiple MS replicas (even)": { extraConfig: func(c *Server) *Server { - return c.WithAccessPoints("1.2.3.4:1234", "5.6.7.8:5678") + return c.WithMgmtSvcReplicas("1.2.3.4:1234", "5.6.7.8:5678") }, - expErr: FaultConfigEvenAccessPoints, + expErr: FaultConfigEvenMgmtSvcReplicas, }, - "multiple access points (odd)": { + "multiple MS replicas (odd)": { extraConfig: func(c *Server) *Server { - return c.WithAccessPoints("1.2.3.4:1234", "5.6.7.8:5678", "1.5.3.8:6247") + return c.WithMgmtSvcReplicas("1.2.3.4:1234", "5.6.7.8:5678", "1.5.3.8:6247") }, }, - "multiple access points (dupes)": { + "multiple MS replicas (dupes)": { extraConfig: func(c *Server) *Server { - return c.WithAccessPoints("1.2.3.4", "5.6.7.8", "1.2.3.4") + return c.WithMgmtSvcReplicas("1.2.3.4", "5.6.7.8", "1.2.3.4") }, - expErr: FaultConfigBadAccessPoints, + expErr: FaultConfigBadMgmtSvcReplicas, }, - "multiple access points (dupes with ports)": { + "multiple MS replicas (dupes with ports)": { extraConfig: func(c *Server) *Server { - return c.WithAccessPoints("1.2.3.4:1234", "5.6.7.8:5678", "1.2.3.4:1234") + return c.WithMgmtSvcReplicas("1.2.3.4:1234", "5.6.7.8:5678", "1.2.3.4:1234") }, - expErr: FaultConfigBadAccessPoints, + expErr: FaultConfigBadMgmtSvcReplicas, }, - "multiple access points (dupes with and without ports)": { + "multiple MS replicas (dupes with and without ports)": { extraConfig: func(c *Server) *Server { - return c.WithAccessPoints("1.2.3.4:10001", "5.6.7.8:5678", "1.2.3.4") + return c.WithMgmtSvcReplicas("1.2.3.4:10001", "5.6.7.8:5678", "1.2.3.4") }, - expErr: FaultConfigBadAccessPoints, + expErr: FaultConfigBadMgmtSvcReplicas, }, - "multiple access points (dupes with different ports)": { + "multiple MS replicas (dupes with different ports)": { extraConfig: func(c *Server) *Server { - return c.WithAccessPoints("1.2.3.4:10002", "5.6.7.8:5678", "1.2.3.4") + return c.WithMgmtSvcReplicas("1.2.3.4:10002", "5.6.7.8:5678", "1.2.3.4") }, }, - "no access points": { + "no MS replicas": { extraConfig: func(c *Server) *Server { - return c.WithAccessPoints() + return c.WithMgmtSvcReplicas() }, - expErr: FaultConfigBadAccessPoints, + expErr: FaultConfigBadMgmtSvcReplicas, }, - "single access point no port": { + "single MS replica no port": { extraConfig: func(c *Server) *Server { - return c.WithAccessPoints("1.2.3.4") + return c.WithMgmtSvcReplicas("1.2.3.4") }, }, - "single access point invalid port": { + "single MS replica invalid port": { extraConfig: func(c *Server) *Server { - return c.WithAccessPoints("1.2.3.4"). + return c.WithMgmtSvcReplicas("1.2.3.4"). WithControlPort(0) }, expErr: FaultConfigBadControlPort, }, - "single access point including invalid port (alphanumeric)": { + "single MS replica including invalid port (alphanumeric)": { extraConfig: func(c *Server) *Server { - return c.WithAccessPoints("1.2.3.4:0a0") + return c.WithMgmtSvcReplicas("1.2.3.4:0a0") }, expErr: FaultConfigBadControlPort, }, - "single access point including invalid port (zero)": { + "single MS replica including invalid port (zero)": { extraConfig: func(c *Server) *Server { - return c.WithAccessPoints("1.2.3.4:0") + return c.WithMgmtSvcReplicas("1.2.3.4:0") }, expErr: FaultConfigBadControlPort, }, - "single access point including negative port": { + "single MS replica including negative port": { extraConfig: func(c *Server) *Server { - return c.WithAccessPoints("1.2.3.4:-10002") + return c.WithMgmtSvcReplicas("1.2.3.4:-10002") }, expErr: FaultConfigBadControlPort, }, - "single access point hostname including negative port": { + "single MS replica hostname including negative port": { extraConfig: func(c *Server) *Server { - return c.WithAccessPoints("hostX:-10002") + return c.WithMgmtSvcReplicas("hostX:-10002") }, expErr: FaultConfigBadControlPort, }, @@ -703,8 +710,8 @@ func TestServerConfig_Validation(t *testing.T) { ), ) }, - expConfig: baseCfg(t, testFile). - WithAccessPoints("hostname1:10001"). + expConfig: baseCfg(t, log, testFile). + WithMgmtSvcReplicas("hostname1:10001", "hostname2:10001", "hostname3:10001"). WithControlMetadata(storage.ControlMetadata{ Path: testMetadataDir, DevicePath: "/dev/something", @@ -778,8 +785,8 @@ func TestServerConfig_Validation(t *testing.T) { ), ) }, - expConfig: baseCfg(t, testFile). - WithAccessPoints("hostname1:10001"). + expConfig: baseCfg(t, log, testFile). + WithMgmtSvcReplicas("hostname1:10001", "hostname2:10001", "hostname3:10001"). WithControlMetadata(storage.ControlMetadata{ Path: testMetadataDir, DevicePath: "/dev/something", @@ -863,8 +870,8 @@ func TestServerConfig_Validation(t *testing.T) { ), ) }, - expConfig: baseCfg(t, testFile). - WithAccessPoints("hostname1:10001"). + expConfig: baseCfg(t, log, testFile). + WithMgmtSvcReplicas("hostname1:10001", "hostname2:10001", "hostname3:10001"). WithControlMetadata(storage.ControlMetadata{ Path: testMetadataDir, }). @@ -956,7 +963,7 @@ func TestServerConfig_Validation(t *testing.T) { } // Apply test case changes to basic config - cfg := tt.extraConfig(baseCfg(t, testFile)) + cfg := tt.extraConfig(baseCfg(t, log, testFile)) log.Debugf("baseCfg metadata: %+v", cfg.Metadata) @@ -1165,7 +1172,7 @@ func TestServerConfig_SetNrHugepages(t *testing.T) { defer test.ShowBufferOnFailure(t, buf) // Apply test case changes to basic config - cfg := tc.extraConfig(baseCfg(t, testFile)) + cfg := tc.extraConfig(baseCfg(t, log, testFile)) mi := &common.MemInfo{ HugepageSizeKiB: defHpSizeKb, @@ -1357,7 +1364,7 @@ func TestServerConfig_SetRamdiskSize(t *testing.T) { defer test.ShowBufferOnFailure(t, buf) // Apply test case changes to basic config - cfg := tc.extraConfig(baseCfg(t, testFile)) + cfg := tc.extraConfig(baseCfg(t, log, testFile)) val := tc.memTotBytes / humanize.KiByte if val > math.MaxInt { @@ -1456,7 +1463,7 @@ func replaceFile(t *testing.T, name, oldTxt, newTxt string) { func TestServerConfig_Parsing(t *testing.T) { noopExtra := func(c *Server) *Server { return c } - cfgFromFile := func(t *testing.T, testFile string, matchText, replaceText []string) (*Server, error) { + cfgFromFile := func(t *testing.T, log logging.Logger, testFile string, matchText, replaceText []string) (*Server, error) { t.Helper() if len(matchText) != len(replaceText) { @@ -1470,17 +1477,17 @@ func TestServerConfig_Parsing(t *testing.T) { replaceFile(t, testFile, m, replaceText[i]) } - return mockConfigFromFile(t, testFile) + return mockConfigFromFile(t, log, testFile) } // load a config based on the server config with all options uncommented. - loadFromFile := func(t *testing.T, testDir string, matchText, replaceText []string) (*Server, error) { + loadFromFile := func(t *testing.T, log logging.Logger, testDir string, matchText, replaceText []string) (*Server, error) { t.Helper() defaultConfigFile := filepath.Join(testDir, sConfigUncomment) uncommentServerConfig(t, defaultConfigFile) - return cfgFromFile(t, defaultConfigFile, matchText, replaceText) + return cfgFromFile(t, log, defaultConfigFile, matchText, replaceText) } for name, tt := range map[string]struct { @@ -1593,7 +1600,7 @@ func TestServerConfig_Parsing(t *testing.T) { tt.outTxtList = []string{tt.outTxt} } - config, errParse := loadFromFile(t, testDir, tt.inTxtList, tt.outTxtList) + config, errParse := loadFromFile(t, log, testDir, tt.inTxtList, tt.outTxtList) test.CmpErr(t, tt.expParseErr, errParse) if tt.expParseErr != nil { return diff --git a/src/control/server/mgmt_pool_test.go b/src/control/server/mgmt_pool_test.go index d5a056b8938..0505279a7b4 100644 --- a/src/control/server/mgmt_pool_test.go +++ b/src/control/server/mgmt_pool_test.go @@ -48,6 +48,7 @@ var ( MemRatio: mockMemRatio, }, } + errNotReplica = errors.New("not a MS replica") ) func getPoolLockCtx(t *testing.T, parent context.Context, sysdb poolDatabase, poolUUID uuid.UUID) (*raft.PoolLock, context.Context) { @@ -442,9 +443,9 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { TierBytes: []uint64{100 * humanize.GiByte, 0}, Properties: testPoolLabelProp(), }, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, - "not access point": { + "not MS replica": { mgmtSvc: notAP, targetCount: 8, req: &mgmtpb.PoolCreateReq{ @@ -452,7 +453,7 @@ func TestServer_MgmtSvc_PoolCreate(t *testing.T) { TierBytes: []uint64{100 * humanize.GiByte, 0}, Properties: testPoolLabelProp(), }, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, "dRPC send fails": { targetCount: 8, @@ -873,7 +874,7 @@ func TestServer_MgmtSvc_PoolDestroy(t *testing.T) { drpcResps: []*mockDrpcResponse{ &mockDrpcResponse{ Message: &mgmtpb.ListContResp{}, - Error: errors.New("not an access point"), + Error: errNotReplica, }, }, expDrpcListContReq: &mgmtpb.ListContReq{ @@ -881,7 +882,7 @@ func TestServer_MgmtSvc_PoolDestroy(t *testing.T) { Id: mockUUID, SvcRanks: []uint32{0, 1, 2}, }, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, // Note: evict dRPC fails as no pool service alive, remains in creating state. // getPoolService() returns TryAgain in resp before list-cont dRPC is issued. @@ -1392,11 +1393,11 @@ func TestServer_MgmtSvc_PoolExtend(t *testing.T) { }, "missing superblock": { mgmtSvc: missingSB, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, - "not access point": { + "not MS replica": { mgmtSvc: notAP, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, "dRPC send fails": { expErr: errors.New("send failure"), @@ -1508,11 +1509,11 @@ func TestServer_MgmtSvc_PoolReintegrate(t *testing.T) { }, "missing superblock": { mgmtSvc: missingSB, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, - "not access point": { + "not MS replica": { mgmtSvc: notAP, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, "dRPC send fails": { expErr: errors.New("send failure"), @@ -1624,12 +1625,12 @@ func TestServer_MgmtSvc_PoolExclude(t *testing.T) { "missing superblock": { mgmtSvc: missingSB, req: &mgmtpb.PoolExcludeReq{Id: mockUUID, Rank: 2, TargetIdx: []uint32{1, 2}}, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, - "not access point": { + "not MS replica": { mgmtSvc: notAP, req: &mgmtpb.PoolExcludeReq{Id: mockUUID, Rank: 2, TargetIdx: []uint32{1, 2}}, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, "dRPC send fails": { req: &mgmtpb.PoolExcludeReq{Id: mockUUID, Rank: 2, TargetIdx: []uint32{1, 2}}, @@ -1720,12 +1721,12 @@ func TestServer_MgmtSvc_PoolDrain(t *testing.T) { "missing superblock": { mgmtSvc: missingSB, req: &mgmtpb.PoolDrainReq{Id: mockUUID, Rank: 2, TargetIdx: []uint32{1, 2}}, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, - "not access point": { + "not MS replica": { mgmtSvc: notAP, req: &mgmtpb.PoolDrainReq{Id: mockUUID, Rank: 2, TargetIdx: []uint32{1, 2}}, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, "dRPC send fails": { req: &mgmtpb.PoolDrainReq{Id: mockUUID, Rank: 2, TargetIdx: []uint32{1, 2}}, @@ -1816,12 +1817,12 @@ func TestServer_MgmtSvc_PoolEvict(t *testing.T) { "missing superblock": { mgmtSvc: missingSB, req: &mgmtpb.PoolEvictReq{Id: mockUUID}, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, - "not access point": { + "not MS replica": { mgmtSvc: notAP, req: &mgmtpb.PoolEvictReq{Id: mockUUID}, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, "dRPC send fails": { req: &mgmtpb.PoolEvictReq{Id: mockUUID}, @@ -2339,7 +2340,7 @@ func TestServer_MgmtSvc_PoolQuery(t *testing.T) { req: &mgmtpb.PoolQueryReq{ Id: mockUUID, }, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, "dRPC send fails": { req: &mgmtpb.PoolQueryReq{ @@ -2708,12 +2709,12 @@ func TestServer_MgmtSvc_PoolUpgrade(t *testing.T) { "missing superblock": { mgmtSvc: missingSB, req: &mgmtpb.PoolUpgradeReq{Id: mockUUID}, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, - "not access point": { + "not MS replica": { mgmtSvc: notAP, req: &mgmtpb.PoolUpgradeReq{Id: mockUUID}, - expErr: errors.New("not an access point"), + expErr: errNotReplica, }, "dRPC send fails": { req: &mgmtpb.PoolUpgradeReq{Id: mockUUID}, diff --git a/src/control/server/mgmt_system.go b/src/control/server/mgmt_system.go index 4dab1724b37..e1a60a06d6a 100644 --- a/src/control/server/mgmt_system.go +++ b/src/control/server/mgmt_system.go @@ -132,7 +132,7 @@ func (svc *mgmtSvc) GetAttachInfo(ctx context.Context, req *mgmtpb.GetAttachInfo return resp, nil } -// LeaderQuery returns the system leader and access point replica details. +// LeaderQuery returns the system leader and MS replica details. func (svc *mgmtSvc) LeaderQuery(ctx context.Context, req *mgmtpb.LeaderQueryReq) (*mgmtpb.LeaderQueryResp, error) { if err := svc.checkSystemRequest(req); err != nil { return nil, err diff --git a/src/control/server/server.go b/src/control/server/server.go index bc54d31ef58..fdf7d4ea8e1 100644 --- a/src/control/server/server.go +++ b/src/control/server/server.go @@ -240,7 +240,7 @@ func (srv *server) createServices(ctx context.Context) (err error) { // Create event distribution primitives. srv.pubSub = events.NewPubSub(ctx, srv.log) srv.OnShutdown(srv.pubSub.Close) - srv.evtForwarder = control.NewEventForwarder(rpcClient, srv.cfg.AccessPoints) + srv.evtForwarder = control.NewEventForwarder(rpcClient, srv.cfg.MgmtSvcReplicas) srv.evtLogger = control.NewEventLogger(srv.log) srv.ctlSvc = NewControlService(srv.log, srv.harness, srv.cfg, srv.pubSub, @@ -321,7 +321,7 @@ func (srv *server) initNetwork() error { func (srv *server) createEngine(ctx context.Context, idx int, cfg *engine.Config) (*EngineInstance, error) { // Closure to join an engine instance to a system using control API. joinFn := func(ctxIn context.Context, req *control.SystemJoinReq) (*control.SystemJoinResp, error) { - req.SetHostList(srv.cfg.AccessPoints) + req.SetHostList(srv.cfg.MgmtSvcReplicas) req.SetSystem(srv.cfg.SystemName) req.ControlAddr = srv.ctlAddr diff --git a/src/control/server/server_utils.go b/src/control/server/server_utils.go index 269a5201e30..1e8c9e0c14d 100644 --- a/src/control/server/server_utils.go +++ b/src/control/server/server_utils.go @@ -112,12 +112,12 @@ func getBdevCfgsFromSrvCfg(cfg *config.Server) storage.TierConfigs { func cfgGetReplicas(cfg *config.Server, lookup ipLookupFn) ([]*net.TCPAddr, error) { var dbReplicas []*net.TCPAddr - for _, ap := range cfg.AccessPoints { - apAddr, err := resolveFirstAddr(ap, lookup) + for _, rep := range cfg.MgmtSvcReplicas { + repAddr, err := resolveFirstAddr(rep, lookup) if err != nil { - return nil, config.FaultConfigBadAccessPoints + return nil, config.FaultConfigBadMgmtSvcReplicas } - dbReplicas = append(dbReplicas, apAddr) + dbReplicas = append(dbReplicas, repAddr) } return dbReplicas, nil diff --git a/src/control/server/server_utils_test.go b/src/control/server/server_utils_test.go index 1c8eadb8856..66b2a4595cc 100644 --- a/src/control/server/server_utils_test.go +++ b/src/control/server/server_utils_test.go @@ -636,7 +636,7 @@ func TestServer_prepBdevStorage(t *testing.T) { cfg := config.DefaultServer(). WithFabricProvider("ofi+verbs"). - WithAccessPoints("foo", "bar", "baz") // Suppress redundancy NOTICE log msg + WithMgmtSvcReplicas("foo", "bar", "baz") // Suppress redundancy NOTICE log msg if tc.srvCfgExtra != nil { cfg = tc.srvCfgExtra(cfg) } diff --git a/src/control/server/util_test.go b/src/control/server/util_test.go index 67542c47d3a..cd6869c59b6 100644 --- a/src/control/server/util_test.go +++ b/src/control/server/util_test.go @@ -265,7 +265,7 @@ func newTestMgmtSvc(t *testing.T, log logging.Logger) *mgmtSvc { // newTestMgmtSvcMulti creates a mgmtSvc that contains the requested // number of EngineInstances. If requested, the first instance is -// configured as an access point. +// configured as a MS replica. func newTestMgmtSvcMulti(t *testing.T, log logging.Logger, count int, isAP bool) *mgmtSvc { harness := NewEngineHarness(log) provider := storage.MockProvider(log, 0, nil, nil, nil, nil, nil) diff --git a/src/tests/ftest/config_file_gen.py b/src/tests/ftest/config_file_gen.py index 58f97f3b902..66172cd2201 100755 --- a/src/tests/ftest/config_file_gen.py +++ b/src/tests/ftest/config_file_gen.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -32,7 +32,7 @@ def generate_agent_config(args): common_cfg = CommonConfig(args.group_name, DaosAgentTransportCredentials()) config = DaosAgentYamlParameters(args.agent_file, common_cfg) # Update the configuration file access points - config.other_params.access_points.value = args.node_list.split(",") + config.access_points.value = args.node_list.split(",") return create_config(args, config) @@ -51,8 +51,8 @@ def generate_server_config(args): config.engine_params[0].storage.storage_tiers[0].storage_class.value = "ram" config.engine_params[0].storage.storage_tiers[0].scm_mount.value = "/mnt/daos" config.engine_params[0].storage.storage_tiers[0].scm_size.value = 0 - # Update the configuration file access points - config.other_params.access_points.value = args.node_list.split(",") + # Update the configuration file MS replicas + config.mgmt_svc_replicas.value = args.node_list.split(",") return create_config(args, config) @@ -153,13 +153,13 @@ def main(): action="store", type=str, default=None, - help="comma-separated list of node names to use as the access points") + help="comma-separated list of node names to use as the MS replicas") parser.add_argument( "-p", "--port", action="store", type=int, default=None, - help="the access point port") + help="the MS replica port") parser.add_argument( "-s", "--server_file", action="store", diff --git a/src/tests/ftest/control/config_generate_output.py b/src/tests/ftest/control/config_generate_output.py index b574088cf98..35ce1cf521f 100644 --- a/src/tests/ftest/control/config_generate_output.py +++ b/src/tests/ftest/control/config_generate_output.py @@ -119,11 +119,11 @@ def check_errors(self, errors): if errors: self.fail("\n----- Errors detected! -----\n{}".format("\n".join(errors))) - def verify_access_point(self, host_port_input, failure_expected=None): - """Run with given AP and verify the AP in the output. + def verify_ms_replica(self, host_port_input, failure_expected=None): + """Run with given MS replica and verify the MS replica in the output. Args: - host_port_input (str): Host:Port or just Host. Supports multiple APs + host_port_input (str): Host:Port or just Host. Supports multiple MS replicas that are separated by comma. failure_expected (str): Expected error message. Set it to None if not expecting any error. Defaults to None. @@ -147,17 +147,17 @@ def verify_access_point(self, host_port_input, failure_expected=None): try: result = dmg.config_generate( - access_points=host_port_input, net_provider=self.def_provider) + mgmt_svc_replicas=host_port_input, net_provider=self.def_provider) except CommandFailure as err: errors.append("Unexpected failure! {}".format(err)) if result.exit_status == 0 and failure_expected is None: try: yaml_data = yaml.safe_load(result.stdout) - check["actual"] = yaml_data["access_points"] + check["actual"] = yaml_data["mgmt_svc_replicas"] if sorted(check["expected"]) != sorted(check["actual"]): errors.append( - "Unexpected access point: {} != {}".format( + "Unexpected MS replica: {} != {}".format( check["expected"], check["actual"])) except yaml.YAMLError as error: errors.append("Error loading dmg generated config!: {}".format(error)) @@ -195,7 +195,7 @@ def test_basic_config(self): # 1. Call dmg config generate. result = self.get_dmg_command().config_generate( - access_points="wolf-a", net_provider=self.def_provider) + mgmt_svc_replicas="wolf-a", net_provider=self.def_provider) generated_yaml = yaml.safe_load(result.stdout) errors = [] @@ -275,7 +275,7 @@ def test_tmpfs_scm_config(self): # Call dmg config generate. result = self.get_dmg_command().config_generate( - access_points="wolf-a", net_provider=self.def_provider, use_tmpfs_scm=True, + mgmt_svc_replicas="wolf-a", net_provider=self.def_provider, use_tmpfs_scm=True, control_metadata_path=self.test_dir) if result.exit_status != 0: errors.append("Config generate failed with use_tmpfs_scm = True!") @@ -328,76 +328,77 @@ def test_tmpfs_scm_config(self): self.check_errors(errors) - def test_access_points_single(self): - """Test --access-points with single AP with and without port. + def test_mgmt_svc_replicas_single(self): + """Test --ms-replica with single MS replica with and without port. :avocado: tags=all,full_regression :avocado: tags=hw,large - :avocado: tags=control,dmg_config_generate,access_points - :avocado: tags=ConfigGenerateOutput,test_access_points_single + :avocado: tags=control,dmg_config_generate,mgmt_svc_replicas + :avocado: tags=ConfigGenerateOutput,test_mgmt_svc_replicas_single """ errors = [] # Single AP. - errors.extend(self.verify_access_point("wolf-a")) + errors.extend(self.verify_ms_replica("wolf-a")) # Single AP with a valid port. - errors.extend(self.verify_access_point("wolf-a:12345")) + errors.extend(self.verify_ms_replica("wolf-a:12345")) self.check_errors(errors) - def test_access_points_odd(self): - """Test --access-points with odd number of APs. + def test_mgmt_svc_replicas_odd(self): + """Test --ms-replicas with odd number of MS replicas. :avocado: tags=all,full_regression :avocado: tags=hw,large - :avocado: tags=control,dmg_config_generate,access_points - :avocado: tags=ConfigGenerateOutput,test_access_points_odd + :avocado: tags=control,dmg_config_generate,mgmt_svc_replicas + :avocado: tags=ConfigGenerateOutput,test_mgmt_svc_replicas_odd """ errors = [] # Odd AP. - errors.extend(self.verify_access_point("wolf-a,wolf-b,wolf-c")) + errors.extend(self.verify_ms_replica("wolf-a,wolf-b,wolf-c")) # Odd AP with port. - errors.extend(self.verify_access_point("wolf-a:12345,wolf-b:12345,wolf-c:12345")) + errors.extend(self.verify_ms_replica("wolf-a:12345,wolf-b:12345,wolf-c:12345")) self.check_errors(errors) - def test_access_points_invalid(self): - """Test --access-points with invalid port. + def test_mgmt_svc_replicas_invalid(self): + """Test --ms-replicas with invalid port. :avocado: tags=all,full_regression :avocado: tags=hw,large - :avocado: tags=control,dmg_config_generate,access_points - :avocado: tags=ConfigGenerateOutput,test_access_points_invalid + :avocado: tags=control,dmg_config_generate,mgmt_svc_replicas + :avocado: tags=ConfigGenerateOutput,test_mgmt_svc_replicas_invalid """ errors = [] # Even AP. - errors.extend(self.verify_access_point("wolf-a,wolf-b", "non-odd")) + errors.extend(self.verify_ms_replica("wolf-a,wolf-b", "non-odd")) # Single AP with an invalid port. - errors.extend(self.verify_access_point("wolf-a:abcd", "invalid access point port")) + errors.extend(self.verify_ms_replica("wolf-a:abcd", "invalid MS replica port")) # Odd AP with both valid and invalid port. errors.extend( - self.verify_access_point( - "wolf-a:12345,wolf-b:12345,wolf-c:abcd", "invalid access point port")) + self.verify_ms_replica( + "wolf-a:12345,wolf-b:12345,wolf-c:abcd", "invalid MS replica port")) self.check_errors(errors) - def test_access_points_same_ap_repeated(self): - """Test --access-points with the same APs repeated. + def test_mgmt_svc_replicas_same_ap_repeated(self): + """Test --ms-replicas with the same MS replicas repeated. :avocado: tags=all,full_regression :avocado: tags=hw,large - :avocado: tags=control,dmg_config_generate,access_points - :avocado: tags=ConfigGenerateOutput,test_access_points_same_ap_repeated + :avocado: tags=control,dmg_config_generate,mgmt_svc_replicas + :avocado: tags=ConfigGenerateOutput,test_mgmt_svc_replicas_same_ap_repeated """ errors = [] - errors.extend(self.verify_access_point("wolf-a,wolf-a,wolf-a", "duplicate access points")) + errors.extend(self.verify_ms_replica("wolf-a,wolf-a,wolf-a", + "duplicate MS replica addresses")) self.check_errors(errors) @@ -432,7 +433,7 @@ def test_num_engines(self): # Call dmg config generate --num-engines=<1 to max_engine> for num_engines in range(1, max_engine + 1): result = dmg.config_generate( - access_points="wolf-a", num_engines=num_engines, net_provider=self.def_provider) + mgmt_svc_replicas="wolf-a", num_engines=num_engines, net_provider=self.def_provider) generated_yaml = yaml.safe_load(result.stdout) actual_num_engines = len(generated_yaml["engines"]) @@ -444,7 +445,7 @@ def test_num_engines(self): # Verify that max_engine + 1 fails. result = dmg.config_generate( - access_points="wolf-a", num_engines=max_engine + 1, net_provider=self.def_provider) + mgmt_svc_replicas="wolf-a", num_engines=max_engine + 1, net_provider=self.def_provider) if result.exit_status == 0: errors.append("Host + invalid num engines succeeded with {}!".format(max_engine + 1)) @@ -473,7 +474,7 @@ def test_scm_only(self): # Call dmg config generate with --scm-only=False result = dmg.config_generate( - access_points="wolf-a", scm_only=False, net_provider=self.def_provider) + mgmt_svc_replicas="wolf-a", scm_only=False, net_provider=self.def_provider) if result.exit_status != 0: errors.append("config generate failed with scm_only = False!") generated_yaml = yaml.safe_load(result.stdout) @@ -492,7 +493,7 @@ def test_scm_only(self): # Call dmg config generate with --scm-only=True result = dmg.config_generate( - access_points="wolf-a", scm_only=True, net_provider=self.def_provider) + mgmt_svc_replicas="wolf-a", scm_only=True, net_provider=self.def_provider) if result.exit_status != 0: errors.append("config generate failed with scm_only = True!") generated_yaml = yaml.safe_load(result.stdout) @@ -548,7 +549,7 @@ def test_net_class(self): for num_engines in range(1, ib_count + 1): # dmg config generate should pass. result = dmg.config_generate( - access_points="wolf-a", num_engines=num_engines, net_class="infiniband", + mgmt_svc_replicas="wolf-a", num_engines=num_engines, net_class="infiniband", net_provider=self.def_provider) if result.exit_status != 0: @@ -574,7 +575,7 @@ def test_net_class(self): # Call dmg config generate --num-engines= # --net-class=infiniband. Too many engines. Should fail. result = dmg.config_generate( - access_points="wolf-a", num_engines=ib_count + 1, net_class="infiniband", + mgmt_svc_replicas="wolf-a", num_engines=ib_count + 1, net_class="infiniband", net_provider=self.def_provider) if result.exit_status == 0: msg = "config generate succeeded with --net-class=infiniband num_engines = {}!".format( @@ -593,7 +594,7 @@ def test_net_class(self): for num_engines in range(1, eth_count + 1): # dmg config generate should pass. result = dmg.config_generate( - access_points="wolf-a", num_engines=num_engines, net_class="ethernet", + mgmt_svc_replicas="wolf-a", num_engines=num_engines, net_class="ethernet", net_provider=self.def_provider) if result.exit_status != 0: @@ -619,7 +620,7 @@ def test_net_class(self): # Call dmg config generate --num-engines= # --net-class=ethernet. Too many engines. Should fail. result = dmg.config_generate( - access_points="wolf-a", num_engines=eth_count + 1, net_class="ethernet", + mgmt_svc_replicas="wolf-a", num_engines=eth_count + 1, net_class="ethernet", net_provider=self.def_provider) if result.exit_status == 0: msg = "config generate succeeded with --net-class=ethernet, num_engines = {}!".format( diff --git a/src/tests/ftest/control/config_generate_run.py b/src/tests/ftest/control/config_generate_run.py index f73ae0f5c7e..cb03f7f33c5 100644 --- a/src/tests/ftest/control/config_generate_run.py +++ b/src/tests/ftest/control/config_generate_run.py @@ -55,7 +55,7 @@ def test_config_generate_run(self): self.log_step("Generating server configuration") server_host = self.hostlist_servers[0] result = self.get_dmg_command().config_generate( - access_points=server_host, num_engines=num_engines, scm_only=scm_only, + mgmt_svc_replicas=server_host, num_engines=num_engines, scm_only=scm_only, net_class=net_class, net_provider=net_provider, use_tmpfs_scm=use_tmpfs_scm, control_metadata_path=control_metadata) diff --git a/src/tests/ftest/control/config_generate_run.yaml b/src/tests/ftest/control/config_generate_run.yaml index 73a1476a0b4..2e27ee1a982 100644 --- a/src/tests/ftest/control/config_generate_run.yaml +++ b/src/tests/ftest/control/config_generate_run.yaml @@ -24,7 +24,7 @@ dmg: setup: start_servers_once: False config_generate_params: !mux - # 1. Access points only. Use default for others. + # 1. MS replicas only. Use default for others. all_default: net_provider: ofi+tcp # 2. Use one engine. diff --git a/src/tests/ftest/control/daos_control_config.py b/src/tests/ftest/control/daos_control_config.py index 2757816102e..c2d5b61c1c0 100644 --- a/src/tests/ftest/control/daos_control_config.py +++ b/src/tests/ftest/control/daos_control_config.py @@ -1,5 +1,5 @@ """ - (C) Copyright 2020-2023 Intel Corporation. + (C) Copyright 2020-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -43,7 +43,7 @@ def test_daos_control_config_basic(self): "Error setting the '{}' config file parameter to '{}'".format( c_val[0], c_val[1])) - # Setup the access points with the server hosts + # Setup the hostlist with the server hosts self.log.info( "Executing dmg config with %s = %s, expecting to %s", c_val[0], c_val[1], c_val[2]) diff --git a/src/tests/ftest/control/daos_system_query.py b/src/tests/ftest/control/daos_system_query.py index 52d9c50877e..89b81cd35ab 100644 --- a/src/tests/ftest/control/daos_system_query.py +++ b/src/tests/ftest/control/daos_system_query.py @@ -32,8 +32,8 @@ def test_daos_system_query(self): exp_sys_name = self.server_managers[0].get_config_value("name") exp_provider = self.server_managers[0].get_config_value("provider") - num_access_points = len(self.host_info.access_points) - exp_num_ap_ranks = num_access_points * engines_per_host + num_ms_replicas = len(self.host_info.mgmt_svc_replicas) + exp_num_ms_ranks = num_ms_replicas * engines_per_host query_output = daos_cmd.system_query()["response"] @@ -50,6 +50,6 @@ def test_daos_system_query(self): self.fail("expected {} rank URIs, got '{}'".format(exp_num_ranks, num_ranks)) num_ap_ranks = len(query_output["access_point_rank_uris"]) - if num_ap_ranks != exp_num_ap_ranks: - self.fail("expected {} access point rank URIs, got '{}'".format(exp_num_ap_ranks, + if num_ap_ranks != exp_num_ms_ranks: + self.fail("expected {} access point rank URIs, got '{}'".format(exp_num_ms_ranks, num_ap_ranks)) diff --git a/src/tests/ftest/control/ms_failover.py b/src/tests/ftest/control/ms_failover.py index 92c668aa607..58fa5cb9e37 100644 --- a/src/tests/ftest/control/ms_failover.py +++ b/src/tests/ftest/control/ms_failover.py @@ -1,5 +1,5 @@ """ -(C) Copyright 2021-2023 Intel Corporation. +(C) Copyright 2021-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -47,7 +47,7 @@ def verify_leader(self, replicas): """Verify the leader of the MS is in the replicas. Args: - replicas (list): list of hostnames representing the access points + replicas (list): list of hostnames representing the replicas for the MS. Returns: @@ -58,7 +58,7 @@ def verify_leader(self, replicas): start = time.time() while not l_hostname and (time.time() - start) < self.L_QUERY_TIMER: l_hostname = self.get_leader() - # Check that the new leader is in the access list + # Check that the new leader is in the MS replica list if l_hostname not in replicas: self.log.error("Selected leader <%s> is not within the replicas" " provided to servers", l_hostname) @@ -78,7 +78,7 @@ def launch_servers(self, replica_count=5): replica_count (int): Number of replicas to launch. Returns: - list: list of access point hosts where MS has been started. + list: list of replica hosts where MS has been started. """ self.log.info("*** launching %d servers", replica_count) @@ -87,7 +87,7 @@ def launch_servers(self, replica_count=5): self.server_group: { "hosts": self.hostlist_servers, - "access_points": replicas, + "mgmt_svc_replicas": replicas, "svr_config_file": None, "dmg_config_file": None, "svr_config_temp": None, diff --git a/src/tests/ftest/control/ms_resilience.py b/src/tests/ftest/control/ms_resilience.py index 8c646a40dbc..61e1f2701c1 100644 --- a/src/tests/ftest/control/ms_resilience.py +++ b/src/tests/ftest/control/ms_resilience.py @@ -1,5 +1,5 @@ """ -(C) Copyright 2021-2023 Intel Corporation. +(C) Copyright 2021-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -99,7 +99,7 @@ def verify_leader(self, replicas): """Verify the leader of the MS is in the replicas. Args: - replicas (NodeSet): host names representing the access points for the MS. + replicas (NodeSet): host names representing the replicas for the MS. Returns: NodeSet: hostname of the MS leader. @@ -165,7 +165,7 @@ def launch_servers(self, resilience_num): resiliency. Returns: - NodeSet: access point hosts where MS has been started. + NodeSet: replica hosts where MS has been started. """ self.log.info("*** launching %d servers", resilience_num) @@ -175,7 +175,7 @@ def launch_servers(self, resilience_num): self.server_group: { "hosts": self.hostlist_servers, - "access_points": replicas, + "mgmt_svc_replicas": replicas, "svr_config_file": None, "dmg_config_file": None, "svr_config_temp": None, diff --git a/src/tests/ftest/harness/config.py b/src/tests/ftest/harness/config.py index e6237c26e0b..4b93643e883 100644 --- a/src/tests/ftest/harness/config.py +++ b/src/tests/ftest/harness/config.py @@ -1,5 +1,5 @@ """ -(C) Copyright 2021-2022 Intel Corporation. +(C) Copyright 2021-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -19,7 +19,7 @@ def test_harness_config(self): """Verify the config handling. Verifies the following: - TestWithServers.access_points + TestWithServers.mgmt_svc_replicas DaosAgentYamlParameters.exclude_fabric_ifaces :avocado: tags=all @@ -27,39 +27,41 @@ def test_harness_config(self): :avocado: tags=harness :avocado: tags=HarnessConfigTest,test_harness_config """ - self.log.info('Verify access_points_suffix set from yaml') - access_points_suffix = self.params.get("access_points_suffix", "/run/setup/*") - self.assertEqual(self.access_points_suffix, access_points_suffix) + self.log.info('Verify mgmt_svc_replicas_suffix set from yaml') + mgmt_svc_replicas_suffix = self.params.get("mgmt_svc_replicas_suffix", "/run/setup/*") + self.assertEqual(self.mgmt_svc_replicas_suffix, mgmt_svc_replicas_suffix) - self.log.info('Verify access_points_suffix is appended exactly once') - access_points = nodeset_append_suffix(self.access_points, access_points_suffix) - self.assertEqual(sorted(self.access_points), sorted(access_points)) - access_points = nodeset_append_suffix(access_points, access_points_suffix) - self.assertEqual(sorted(self.access_points), sorted(access_points)) + self.log.info('Verify mgmt_svc_replicas_suffix is appended exactly once') + mgmt_svc_replicas = nodeset_append_suffix(self.mgmt_svc_replicas, mgmt_svc_replicas_suffix) + self.assertEqual(sorted(self.mgmt_svc_replicas), sorted(mgmt_svc_replicas)) + mgmt_svc_replicas = nodeset_append_suffix(mgmt_svc_replicas, mgmt_svc_replicas_suffix) + self.assertEqual(sorted(self.mgmt_svc_replicas), sorted(mgmt_svc_replicas)) self.log.info('Verify self.get_dmg_command().hostlist_suffix and hostlist') dmg = self.get_dmg_command() - self.assertEqual(dmg.hostlist_suffix, self.access_points_suffix) - expected_hostlist = sorted(nodeset_append_suffix(dmg.hostlist, self.access_points_suffix)) + self.assertEqual(dmg.hostlist_suffix, self.mgmt_svc_replicas_suffix) + expected_hostlist = sorted(nodeset_append_suffix(dmg.hostlist, + self.mgmt_svc_replicas_suffix)) self.assertEqual(sorted(dmg.hostlist), expected_hostlist) self.log.info('Verify self.get_dmg_command().yaml.get_yaml_data()["hostlist"]') self.assertEqual(sorted(dmg.yaml.get_yaml_data()['hostlist']), expected_hostlist) self.log.info('Verify DmgCommand().hostlist_suffix and hostlist') - dmg2 = DmgCommand(self.bin, hostlist_suffix=access_points_suffix) + dmg2 = DmgCommand(self.bin, hostlist_suffix=mgmt_svc_replicas_suffix) dmg2.hostlist = dmg.hostlist - self.assertEqual(dmg2.hostlist_suffix, self.access_points_suffix) + self.assertEqual(dmg2.hostlist_suffix, self.mgmt_svc_replicas_suffix) self.assertEqual(sorted(dmg2.hostlist), expected_hostlist) - self.log.info('Verify server_manager...get_yaml_data...access_points"]') + self.log.info('Verify server_manager...get_yaml_data...mgmt_svc_replicas"]') yaml_data = self.server_managers[0].manager.job.yaml.get_yaml_data() - self.assertEqual(sorted(yaml_data['access_points']), sorted(self.access_points)) + self.assertEqual(sorted(yaml_data['mgmt_svc_replicas']), sorted(self.mgmt_svc_replicas)) - self.log.info('Verify daos_server.yaml access_points') + self.log.info('Verify daos_server.yaml mgmt_svc_replicas') with open(self.server_managers[0].manager.job.temporary_file, 'r') as yaml_file: daos_server_yaml = yaml.safe_load(yaml_file.read()) - self.assertEqual(sorted(daos_server_yaml['access_points']), sorted(self.access_points)) + self.assertEqual(sorted(daos_server_yaml['mgmt_svc_replicas']), + sorted(self.mgmt_svc_replicas)) self.log.info('Verify daos_control.yaml hostlist') with open(self.get_dmg_command().temporary_file, 'r') as yaml_file: @@ -69,7 +71,7 @@ def test_harness_config(self): self.log.info('Verify daos_agent.yaml access_points') with open(self.agent_managers[0].manager.job.temporary_file, 'r') as yaml_file: daos_agent_yaml = yaml.safe_load(yaml_file.read()) - self.assertEqual(sorted(daos_agent_yaml['access_points']), sorted(self.access_points)) + self.assertEqual(sorted(daos_agent_yaml['access_points']), sorted(self.mgmt_svc_replicas)) self.log.info('Verify daos_agent.yaml exclude_fabric_ifaces') expected = self.params.get('exclude_fabric_ifaces', '/run/agent_config/*') diff --git a/src/tests/ftest/harness/config.yaml b/src/tests/ftest/harness/config.yaml index be19a641812..4c2f98a7572 100644 --- a/src/tests/ftest/harness/config.yaml +++ b/src/tests/ftest/harness/config.yaml @@ -3,7 +3,7 @@ hosts: test_clients: 1 timeout: 60 setup: - access_points_suffix: .wolf.hpdd.intel.com + mgmt_svc_replicas_suffix: .wolf.hpdd.intel.com server_config: name: daos_server engines_per_host: 1 diff --git a/src/tests/ftest/network/cart_self_test.py b/src/tests/ftest/network/cart_self_test.py index 9dbf0f5e884..afb74c142e7 100644 --- a/src/tests/ftest/network/cart_self_test.py +++ b/src/tests/ftest/network/cart_self_test.py @@ -62,7 +62,7 @@ def setUp(self): self.server_managers[-1], self.hostlist_servers, self.hostfile_servers_slots, - self.access_points) + self.mgmt_svc_replicas) # Setup additional environment variables for the server orterun command self.cart_env["CRT_CTX_NUM"] = "8" diff --git a/src/tests/ftest/pool/destroy.py b/src/tests/ftest/pool/destroy.py index e5dcb6cc521..d9a70534f95 100644 --- a/src/tests/ftest/pool/destroy.py +++ b/src/tests/ftest/pool/destroy.py @@ -1,5 +1,5 @@ """ -(C) Copyright 2018-2023 Intel Corporation. +(C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -53,13 +53,13 @@ def get_group_info(hosts, svr_config_file=None, dmg_config_file=None, using the config_file specification. Defaults to None. Returns: - dict: a dictionary identifying the hosts and access points for the + dict: a dictionary identifying the hosts and MS replicas for the server group dictionary """ return { "hosts": hosts, - "access_points": hosts[:1], + "mgmt_svc_replicas": hosts[:1], "svr_config_file": svr_config_file, "dmg_config_file": dmg_config_file, "svr_config_temp": svr_config_temp, diff --git a/src/tests/ftest/pool/destroy_rebuild.py b/src/tests/ftest/pool/destroy_rebuild.py index 753e8fb009e..bb4c48ed8df 100644 --- a/src/tests/ftest/pool/destroy_rebuild.py +++ b/src/tests/ftest/pool/destroy_rebuild.py @@ -1,5 +1,5 @@ ''' - (C) Copyright 2018-2023 Intel Corporation. + (C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent ''' @@ -54,27 +54,27 @@ def test_destroy_while_rebuilding(self): # 3. self.log_step("Start rebuild, system stop") all_ranks = self.server_managers[0].ranks.keys() - ap_ranks = self.server_managers[0].get_host_ranks(self.access_points) - non_ap_ranks = list(set(all_ranks) - set(ap_ranks)) + ms_ranks = self.server_managers[0].get_host_ranks(self.mgmt_svc_replicas) + non_ms_ranks = list(set(all_ranks) - set(ms_ranks)) # Get the pool leader rank pool.set_query_data() leader_rank = pool.query_data["response"]["svc_ldr"] - if leader_rank in ap_ranks: - ap_ranks.remove(leader_rank) - elif leader_rank in non_ap_ranks: - non_ap_ranks.remove(leader_rank) + if leader_rank in ms_ranks: + ms_ranks.remove(leader_rank) + elif leader_rank in non_ms_ranks: + non_ms_ranks.remove(leader_rank) # Select the following ranks to stop # - the pool leader rank - # - a random rank that is not an access point - # - a random rank this is an access point and not the pool leader + # - a random rank that is not a MS replica + # - a random rank this is a MS replica and not the pool leader self.log.debug( - "Engine ranks: pool leader=%s, access points=%s, other=%s", - leader_rank, ap_ranks, non_ap_ranks) + "Engine ranks: pool leader=%s, MS replicas=%s, other=%s", + leader_rank, ms_ranks, non_ms_ranks) ranks = [leader_rank] - ranks.append(random.choice(ap_ranks)) # nosec - ranks.append(random.choice(non_ap_ranks)) # nosec + ranks.append(random.choice(ms_ranks)) # nosec + ranks.append(random.choice(non_ms_ranks)) # nosec self.log.info("ranks to rebuild: %s", ranks) self.server_managers[0].stop_ranks(ranks, self.d_log, force=True) diff --git a/src/tests/ftest/pool/destroy_rebuild.yaml b/src/tests/ftest/pool/destroy_rebuild.yaml index e927f7357e5..f599737977b 100644 --- a/src/tests/ftest/pool/destroy_rebuild.yaml +++ b/src/tests/ftest/pool/destroy_rebuild.yaml @@ -5,7 +5,7 @@ hosts: test_servers: 7 test_clients: 1 setup: - access_points_qty: 5 + mgmt_svc_replicas_qty: 5 server_config: name: daos_server engines_per_host: 1 diff --git a/src/tests/ftest/server/daos_server_config.yaml b/src/tests/ftest/server/daos_server_config.yaml index 0f6716c46ba..429789fcdc7 100644 --- a/src/tests/ftest/server/daos_server_config.yaml +++ b/src/tests/ftest/server/daos_server_config.yaml @@ -44,24 +44,24 @@ server_config_val: !mux - "name" - "amjustin, 12345, abcdef" - "FAIL" - access_point_noport: + mgt_svc_replica_noport: config_val: - - "access_points" + - "mgmt_svc_replicas" - [localhost] - "PASS" - # access_point_repeated: + # mgmt_svc_replica_repeated: # config_val: - # - "access_points" + # - "mgmt_svc_replicas" # - [localhost:10001, localhost:10001] # - "FAIL" - # access_point_wrong_port: + # mgmt_svc_replica_wrong_port: # config_val: - # - "access_points" + # - "mgmt_svc_replicas" # - [localhost:12345] # - "FAIL" - # access_point_wrong_name: + # mgmt_svc_replica_wrong_name: # config_val: - # - "access_points" + # - "mgmt_svc_replicas" # - [wrongName:10001] # - "FAIL" control_log_file_does_not_exist: diff --git a/src/tests/ftest/util/agent_utils_params.py b/src/tests/ftest/util/agent_utils_params.py index 9fe4ddd455e..bd091a162e6 100644 --- a/src/tests/ftest/util/agent_utils_params.py +++ b/src/tests/ftest/util/agent_utils_params.py @@ -72,6 +72,10 @@ def __init__(self, filename, common_yaml): # Enable client telemetry for all client processes. # - telemetry_retain: , e.g. 5m # Time to retain per-client telemetry data. + # - access_points: , e.g. ["hostname1:10001"] + # Hosts can be specified with or without port, default port below + # assumed if not specified. Defaults to the hostname of this node + # at port 10000 for local testing. self.runtime_dir = BasicParameter(None, default_runtime_dir) self.log_file = LogParameter(log_dir, None, "daos_agent.log") self.control_log_mask = BasicParameter(None, "debug") @@ -81,6 +85,7 @@ def __init__(self, filename, common_yaml): self.telemetry_port = BasicParameter(None) self.telemetry_enabled = BasicParameter(None) self.telemetry_retain = BasicParameter(None) + self.access_points = BasicParameter(None, ["localhost"]) def update_log_file(self, name): """Update the log file name for the daos agent. diff --git a/src/tests/ftest/util/apricot/apricot/test.py b/src/tests/ftest/util/apricot/apricot/test.py index bb4dba4f1e0..185849c0189 100644 --- a/src/tests/ftest/util/apricot/apricot/test.py +++ b/src/tests/ftest/util/apricot/apricot/test.py @@ -25,8 +25,9 @@ from environment_utils import TestEnvironment from exception_utils import CommandFailure from fault_config_utils import FaultInjection -from general_utils import (dict_to_str, dump_engines_stacks, get_avocado_config_value, - nodeset_append_suffix, set_avocado_config_value) +from general_utils import (DaosTestError, dict_to_str, dump_engines_stacks, + get_avocado_config_value, nodeset_append_suffix, + set_avocado_config_value) from host_utils import HostException, HostInfo, HostRole, get_host_parameters, get_local_host from logger_utils import TestLogger from pydaos.raw import DaosApiError, DaosContext, DaosLog @@ -665,8 +666,8 @@ def __init__(self, *args, **kwargs): self.__dump_engine_ult_on_failure = True # Whether engines ULT stacks have been already dumped self.__have_dumped_ult_stacks = False - # Suffix to append to each access point name - self.access_points_suffix = None + # Suffix to append to each MS replica name + self.mgmt_svc_replicas_suffix = None def setUp(self): """Set up each test case.""" @@ -723,23 +724,23 @@ def setUp(self): self.hostlist_servers = NodeSet(self.host_info.servers.hosts) self.hostlist_clients = NodeSet(self.host_info.clients.hosts) - # Access points to use by default when starting servers and agents - # - for 1 or 2 servers use 1 access point - # - for 3 or more servers use 3 access points - default_access_points_qty = 1 if len(self.hostlist_servers) < 3 else 3 - access_points_qty = self.params.get( - "access_points_qty", "/run/setup/*", default_access_points_qty) - if access_points_qty < 1 or access_points_qty > len(self.hostlist_servers): - self.fail("Invalid access points node quantity") - default_access_points = self.hostlist_servers[:access_points_qty] - self.access_points = NodeSet( - self.params.get("access_points", "/run/setup/*", default_access_points)) - self.access_points_suffix = self.params.get( - "access_points_suffix", "/run/setup/*", self.access_points_suffix) - if self.access_points_suffix: - self.access_points = nodeset_append_suffix( - self.access_points, self.access_points_suffix) - self.host_info.access_points = self.access_points + # MS replicas to use by default when starting servers and agents + # - for 1 or 2 servers use 1 replica + # - for 3 or more servers use 3 replicas + default_mgmt_svc_replicas_qty = 1 if len(self.hostlist_servers) < 3 else 3 + mgmt_svc_replicas_qty = self.params.get( + "mgmt_svc_replicas_qty", "/run/setup/*", default_mgmt_svc_replicas_qty) + if mgmt_svc_replicas_qty < 1 or mgmt_svc_replicas_qty > len(self.hostlist_servers): + self.fail("Invalid MS replica node quantity") + default_mgmt_svc_replicas = self.hostlist_servers[:mgmt_svc_replicas_qty] + self.mgmt_svc_replicas = NodeSet( + self.params.get("mgmt_svc_replicas", "/run/setup/*", default_mgmt_svc_replicas)) + self.mgmt_svc_replicas_suffix = self.params.get( + "mgmt_svc_replicas_suffix", "/run/setup/*", self.mgmt_svc_replicas_suffix) + if self.mgmt_svc_replicas_suffix: + self.mgmt_svc_replicas = nodeset_append_suffix( + self.mgmt_svc_replicas, self.mgmt_svc_replicas_suffix) + self.host_info.mgmt_svc_replicas = self.mgmt_svc_replicas # Toggle whether to dump server ULT stacks on failure self.__dump_engine_ult_on_failure = self.params.get( @@ -880,9 +881,9 @@ def start_agents(self, agent_groups=None, force=False): Args: agent_groups (dict, optional): dictionary of dictionaries, containing the list of hosts on which to start the daos agent - and the list of server access points, using a unique server + and the list of MS replicas, using a unique server group name key. Defaults to None which will use the server group - name, all of the client hosts, and the access points from the + name, all of the client hosts, and the MS replicas from the test's yaml file to define a single server group entry. force (bool, optional): whether or not to force starting the agents. Defaults to False. @@ -903,9 +904,9 @@ def start_servers(self, server_groups=None, force=False): Args: server_groups (dict, optional): dictionary of dictionaries, containing the list of hosts on which to start the daos server - and the list of access points, using a unique server group name + and the list of MS replicas, using a unique server group name key. Defaults to None which will use the server group name, all - of the server hosts, and the access points from the test's yaml + of the server hosts, and the MS replicas from the test's yaml file to define a single server group entry. force (bool, optional): whether or not to force starting the servers. Defaults to False. @@ -970,9 +971,9 @@ def setup_agents(self, agent_groups=None): Args: agent_groups (dict, optional): dictionary of dictionaries, containing the list of hosts on which to start the daos agent - and the list of server access points, using a unique server + and the list of server MS replicas, using a unique server group name key. Defaults to None which will use the server group - name, all of the client hosts, and the access points from the + name, all of the client hosts, and the MS replicas from the test's yaml file to define a single server group entry. Raises: @@ -986,7 +987,7 @@ def setup_agents(self, agent_groups=None): agent_groups = { self.server_group: { "hosts": include_local_host(self.hostlist_clients), - "access_points": self.access_points + "mgmt_svc_replicas": self.mgmt_svc_replicas } } @@ -1001,7 +1002,7 @@ def setup_agents(self, agent_groups=None): self.agent_managers[-1], info["hosts"], self.hostfile_clients_slots, - info["access_points"]) + info["mgmt_svc_replicas"]) def setup_servers(self, server_groups=None): """Start the daos_server processes. @@ -1009,9 +1010,9 @@ def setup_servers(self, server_groups=None): Args: server_groups (dict, optional): dictionary of dictionaries, containing the list of hosts on which to start the daos server - and the list of access points, using a unique server group name + and the list of MS replicas, using a unique server group name key. Defaults to None which will use the server group name, all - of the server hosts, and the access points from the test's yaml + of the server hosts, and the MS replicas from the test's yaml file to define a single server group entry. Raises: @@ -1023,7 +1024,7 @@ def setup_servers(self, server_groups=None): server_groups = { self.server_group: { "hosts": self.hostlist_servers, - "access_points": self.access_points, + "mgmt_svc_replicas": self.mgmt_svc_replicas, "svr_config_file": None, "dmg_config_file": None, "svr_config_temp": None, @@ -1044,7 +1045,7 @@ def setup_servers(self, server_groups=None): self.server_managers[-1], info["hosts"], self.hostfile_servers_slots, - info["access_points"]) + info["mgmt_svc_replicas"]) def get_config_file(self, name, command, path=None): """Get the yaml configuration file. @@ -1164,7 +1165,7 @@ def add_server_manager(self, group=None, svr_config_file=None, DaosServerManager( group, self.bin, svr_cert_dir, svr_config_file, dmg_cert_dir, dmg_config_file, svr_config_temp, dmg_config_temp, - self.server_manager_class, access_points_suffix=self.access_points_suffix) + self.server_manager_class, mgmt_svc_replicas_suffix=self.mgmt_svc_replicas_suffix) ) if self.server_config_namespace is not None: self.log.debug( @@ -1172,7 +1173,7 @@ def add_server_manager(self, group=None, svr_config_file=None, len(self.server_managers) - 1, self.server_config_namespace) self.server_managers[-1].manager.job.yaml.namespace = self.server_config_namespace - def configure_manager(self, name, manager, hosts, slots, access_points=None): + def configure_manager(self, name, manager, hosts, slots, mgmt_svc_replicas=None): """Configure the agent/server manager object. Defines the environment variables, host list, and hostfile settings used @@ -1183,17 +1184,22 @@ def configure_manager(self, name, manager, hosts, slots, access_points=None): manager (SubprocessManager): the daos agent/server process manager hosts (NodeSet): hosts on which to start the daos agent/server slots (int): number of slots per engine to define in the hostfile - access_points (NodeSet): access point hosts. Defaults to None which - uses self.access_points. + mgmt_svc_replicas (NodeSet): MS replica hosts. Defaults to None which + uses self.mgmt_svc_replicas. """ self.log.info("-" * 100) self.log.info("--- CONFIGURING %s MANAGER ---", name.upper()) - if access_points is None: - access_points = NodeSet(self.access_points) + if mgmt_svc_replicas is None: + mgmt_svc_replicas = NodeSet(self.mgmt_svc_replicas) # Calling get_params() will set the test-specific log names manager.get_params(self) - manager.set_config_value("access_points", list(access_points)) + if name == "server": + manager.set_config_value("mgmt_svc_replicas", list(mgmt_svc_replicas)) + elif name == "agent": + manager.set_config_value("access_points", list(mgmt_svc_replicas)) + else: + raise DaosTestError("invalid manager name: {}".format(name)) manager.manager.assign_environment( EnvironmentVariables({"PATH": None}), True) manager.hosts = (hosts, self.workdir, slots) @@ -1653,13 +1659,13 @@ def get_dmg_command(self, index=0): """Get a DmgCommand setup to interact with server manager index. Return a DmgCommand object configured with: - - the "-l" parameter assigned to the server's access point list + - the "-l" parameter assigned to the server's MS replica list - the "-i" parameter assigned to the server's interactive mode This method is intended to be used by tests that wants to use dmg to create and destroy pool. Pass in the object to TestPool constructor. - Access point should be passed in to -l regardless of the number of + MS replica should be passed in to -l regardless of the number of servers. Args: @@ -1683,8 +1689,8 @@ def get_dmg_command(self, index=0): dmg_cmd = get_dmg_command( self.server_group, dmg_cert_dir, self.bin, dmg_config_file, - dmg_config_temp, self.access_points_suffix) - dmg_cmd.hostlist = self.access_points + dmg_config_temp, self.mgmt_svc_replicas_suffix) + dmg_cmd.hostlist = self.mgmt_svc_replicas return dmg_cmd def get_daos_command(self): @@ -1829,7 +1835,7 @@ def add_container_qty(self, quantity, pool, namespace=CONT_NAMESPACE, create=Tru self.container.append( self.get_container(pool=pool, namespace=namespace, create=create)) - def start_additional_servers(self, additional_servers, index=0, access_points=None): + def start_additional_servers(self, additional_servers, index=0, mgmt_svc_replicas=None): """Start additional servers. This method can be used to start a new daos_server during a test. @@ -1838,8 +1844,8 @@ def start_additional_servers(self, additional_servers, index=0, access_points=No additional_servers (NodeSet): hosts on which to start daos_server. index (int): Determines which server_managers to use when creating the new server. - access_points (NodeSet): access point hosts. Defaults to None which - uses self.access_points. + mgmt_svc_replicas (NodeSet): MS replica hosts. Defaults to None which + uses self.mgmt_svc_replicas. """ self.add_server_manager( self.server_managers[index].manager.job.get_config_value("name"), @@ -1853,6 +1859,6 @@ def start_additional_servers(self, additional_servers, index=0, access_points=No self.server_managers[-1], additional_servers, self.hostfile_servers_slots, - access_points + mgmt_svc_replicas ) self._start_manager_list("server", [self.server_managers[-1]]) diff --git a/src/tests/ftest/util/command_utils_base.py b/src/tests/ftest/util/command_utils_base.py index 837e61b339f..d867fbba4c8 100644 --- a/src/tests/ftest/util/command_utils_base.py +++ b/src/tests/ftest/util/command_utils_base.py @@ -794,7 +794,6 @@ class CommonConfig(YamlParameters): Includes: - the daos system name (name) - - a list of access point nodes (access_points) - the default port number (port) - the transport credentials """ @@ -812,18 +811,12 @@ def __init__(self, name, transport): # - name: , e.g. "daos_server" # Name associated with the DAOS system. # - # - access_points: , e.g. ["hostname1:10001"] - # Hosts can be specified with or without port, default port below - # assumed if not specified. Defaults to the hostname of this node - # at port 10000 for local testing - # # - port: , e.g. 10001 # Default port number with with to bind the daos_server. This # will also be used when connecting to access points if the list # only contains host names. # self.name = BasicParameter(None, name) - self.access_points = BasicParameter(None, ["localhost"]) self.port = BasicParameter(None, 10001) def _get_new(self): diff --git a/src/tests/ftest/util/dmg_utils.py b/src/tests/ftest/util/dmg_utils.py index 881cb298a0a..cbca403895a 100644 --- a/src/tests/ftest/util/dmg_utils.py +++ b/src/tests/ftest/util/dmg_utils.py @@ -1216,13 +1216,13 @@ def pool_evict(self, pool): """ return self._get_result(("pool", "evict"), pool=pool) - def config_generate(self, access_points, num_engines=None, scm_only=False, + def config_generate(self, mgmt_svc_replicas, num_engines=None, scm_only=False, net_class=None, net_provider=None, use_tmpfs_scm=False, control_metadata_path=None): """Produce a server configuration. Args: - access_points (str): Comma separated list of access point addresses. + mgmt_svc_replicas (str): Comma separated list of MS replica addresses. num_pmem (int): Number of SCM (pmem) devices required per storage host in DAOS system. Defaults to None. scm_only (bool, option): Whether to omit NVMe from generated config. @@ -1242,7 +1242,7 @@ def config_generate(self, access_points, num_engines=None, scm_only=False, """ return self._get_result( - ("config", "generate"), access_points=access_points, + ("config", "generate"), mgmt_svc_replicas=mgmt_svc_replicas, num_engines=num_engines, scm_only=scm_only, net_class=net_class, net_provider=net_provider, use_tmpfs_scm=use_tmpfs_scm, control_metadata_path=control_metadata_path) diff --git a/src/tests/ftest/util/dmg_utils_base.py b/src/tests/ftest/util/dmg_utils_base.py index 7e5d2300a53..4787c05d393 100644 --- a/src/tests/ftest/util/dmg_utils_base.py +++ b/src/tests/ftest/util/dmg_utils_base.py @@ -235,7 +235,7 @@ def __init__(self): super( DmgCommandBase.ConfigSubCommand.GenerateSubCommand, self).__init__("/run/dmg/config/generate/*", "generate") - self.access_points = FormattedParameter("--access-points={}", None) + self.mgmt_svc_replicas = FormattedParameter("--ms-replicas={}", None) self.num_engines = FormattedParameter("--num-engines={}", None) self.scm_only = FormattedParameter("--scm-only", False) self.net_class = FormattedParameter("--net-class={}", None) diff --git a/src/tests/ftest/util/host_utils.py b/src/tests/ftest/util/host_utils.py index 83add3924c2..5277321df4a 100644 --- a/src/tests/ftest/util/host_utils.py +++ b/src/tests/ftest/util/host_utils.py @@ -1,5 +1,5 @@ """ -(C) Copyright 2018-2023 Intel Corporation. +(C) Copyright 2018-2024 Intel Corporation. SPDX-License-Identifier: BSD-2-Clause-Patent """ @@ -75,7 +75,7 @@ def __init__(self): """Initialize a HostInfo object.""" self._servers = HostRole() self._clients = HostRole() - self.access_points = NodeSet() + self.mgmt_svc_replicas = NodeSet() @property def all_hosts(self): @@ -119,7 +119,7 @@ def display(self, log): log.info("client_partition: %s", self.clients.partition.name) log.info("server_reservation: %s", self.servers.partition.reservation) log.info("client_reservation: %s", self.clients.partition.reservation) - log.info("access_points: %s", self.access_points) + log.info("mgmt_svc_replicas: %s", self.mgmt_svc_replicas) def set_hosts(self, log, control_host, server_hosts, server_partition, server_reservation, client_hosts, client_partition, client_reservation, include_local_host=False): diff --git a/src/tests/ftest/util/server_utils.py b/src/tests/ftest/util/server_utils.py index 752473021a3..ec79f029c6e 100644 --- a/src/tests/ftest/util/server_utils.py +++ b/src/tests/ftest/util/server_utils.py @@ -71,7 +71,7 @@ class DaosServerManager(SubprocessManager): def __init__(self, group, bin_dir, svr_cert_dir, svr_config_file, dmg_cert_dir, dmg_config_file, svr_config_temp=None, dmg_config_temp=None, manager="Orterun", - namespace="/run/server_manager/*", access_points_suffix=None): + namespace="/run/server_manager/*", mgmt_svc_replicas_suffix=None): # pylint: disable=too-many-arguments """Initialize a DaosServerManager object. @@ -92,7 +92,7 @@ def __init__(self, group, bin_dir, manage the YamlCommand defined through the "job" attribute. Defaults to "Orterun". namespace (str): yaml namespace (path to parameters) - access_points_suffix (str, optional): Suffix to append to each access point name. + mgmt_svc_replicas_suffix (str, optional): Suffix to append to each MS replica name. Defaults to None. """ self.group = group @@ -104,7 +104,8 @@ def __init__(self, group, bin_dir, # Dmg command to access this group of servers which will be configured # to access the daos_servers when they are started self.dmg = get_dmg_command( - group, dmg_cert_dir, bin_dir, dmg_config_file, dmg_config_temp, access_points_suffix) + group, dmg_cert_dir, bin_dir, dmg_config_file, dmg_config_temp, + mgmt_svc_replicas_suffix) # Set the correct certificate file ownership if manager == "Systemctl": @@ -166,7 +167,7 @@ def management_service_hosts(self): NodeSet: the hosts running the management service """ - return NodeSet.fromlist(self.get_config_value('access_points')) + return NodeSet.fromlist(self.get_config_value('mgmt_svc_replicas')) @property def management_service_ranks(self): @@ -202,7 +203,7 @@ def prepare_dmg(self, hosts=None): Args: hosts (list, optional): dmg hostlist value. Defaults to None which - results in using the 'access_points' host list. + results in using the 'mgmt_svc_replicas' host list. """ self._prepare_dmg_certificates() self._prepare_dmg_hostlist(hosts) diff --git a/src/tests/ftest/util/server_utils_params.py b/src/tests/ftest/util/server_utils_params.py index 46db4891220..19dd8ea4df3 100644 --- a/src/tests/ftest/util/server_utils_params.py +++ b/src/tests/ftest/util/server_utils_params.py @@ -107,6 +107,10 @@ def __init__(self, filename, common_yaml): # is set for the running process. If group look up fails or user # is not member, use uid return from user lookup. # + # - mgmt_svc_replicas: , e.g. ["hostname1:10001"] + # Hosts can be specified with or without port, default port below + # assumed if not specified. Defaults to the hostname of this node + # at port 10000 for local testing. default_provider = os.environ.get("D_PROVIDER", "ofi+tcp") # All log files should be placed in the same directory on each host to @@ -133,6 +137,7 @@ def __init__(self, filename, common_yaml): self.helper_log_file = LogParameter(log_dir, None, "daos_server_helper.log") self.telemetry_port = BasicParameter(None, 9191) self.client_env_vars = BasicParameter(None) + self.mgmt_svc_replicas = BasicParameter(None, ["localhost"]) # Used to drop privileges before starting data plane # (if started as root to perform hardware provisioning) diff --git a/src/tests/ftest/util/test_utils_pool.py b/src/tests/ftest/util/test_utils_pool.py index e75510ec8b5..71f05bb131e 100644 --- a/src/tests/ftest/util/test_utils_pool.py +++ b/src/tests/ftest/util/test_utils_pool.py @@ -194,7 +194,7 @@ def __init__(self, context, dmg_command, label_generator=None, namespace=POOL_NA self.context when calling from a test. dmg_command (DmgCommand): DmgCommand used to call dmg command. This value can be obtained by calling self.get_dmg_command() from a - test. It'll return the object with -l + test. It'll return the object with -l and --insecure. label_generator (LabelGenerator, optional): Generates label by adding number to the end of the prefix set in self.label. diff --git a/utils/config/daos_server.yml b/utils/config/daos_server.yml index ea6bf7e5154..a9642631f2d 100644 --- a/utils/config/daos_server.yml +++ b/utils/config/daos_server.yml @@ -15,16 +15,20 @@ #name: daos_server # # -## Access points +## MS replicas ## Immutable after running "dmg storage format". # -## To operate, DAOS will need a quorum of access point nodes to be available. -## Must have the same value for all agents and servers in a system. +## To operate, DAOS requires a quorum of Management Service (MS) replica +## hosts to be available. All servers (replica or otherwise) must have the +## same list of replicas in order for the system to operate correctly. Choose +## 3-5 hosts to serve as replicas, preferably not co-located within the same +## fault domains. +## ## Hosts can be specified with or without port. The default port that is set ## up in port: will be used if a port is not specified here. # ## default: hostname of this node -#access_points: ['hostname1'] +#mgmt_svc_replicas: ['hostname1', 'hostname2', 'hostname3'] # # ## Control plane metadata @@ -46,7 +50,7 @@ ## Default control plane port # ## Port number to bind daos_server to. This will also be used when connecting -## to access points, unless a port is specified in access_points: +## to MS replicas, unless a port is specified in mgmt_svc_replicas: # ## default: 10001 #port: 10001 diff --git a/utils/config/examples/daos_server_local.yml b/utils/config/examples/daos_server_local.yml index 814ac659824..ac5bb6ee808 100644 --- a/utils/config/examples/daos_server_local.yml +++ b/utils/config/examples/daos_server_local.yml @@ -1,7 +1,7 @@ # For a single-server system name: daos_server -access_points: ['localhost'] +mgmt_svc_replicas: ['localhost'] provider: ofi+tcp control_log_file: /tmp/daos_server.log transport_config: diff --git a/utils/config/examples/daos_server_mdonssd.yml b/utils/config/examples/daos_server_mdonssd.yml index 090df281a6b..8b73e53e431 100644 --- a/utils/config/examples/daos_server_mdonssd.yml +++ b/utils/config/examples/daos_server_mdonssd.yml @@ -1,8 +1,11 @@ # Example configuration file for Metadata on SSD. -name: daos_server # sys group daos_server -access_points: ['example'] # management service leader (bootstrap) -# port: 10001 # control listen port, default 10001 +# sys group daos_server +name: daos_server +# management service replicas +mgmt_svc_replicas: ['example1', 'example2', 'example3'] +# control listen port, default 10001 +# port: 10001 provider: ofi+tcp control_log_mask: INFO control_log_file: /tmp/daos_server.log diff --git a/utils/config/examples/daos_server_tcp.yml b/utils/config/examples/daos_server_tcp.yml index 635abe89dce..38f40d7ec67 100644 --- a/utils/config/examples/daos_server_tcp.yml +++ b/utils/config/examples/daos_server_tcp.yml @@ -1,8 +1,11 @@ # Example configuration file using TCP sockets -name: daos_server # sys group daos_server -access_points: ['example'] # management service leader (bootstrap) -# port: 10001 # control listen port, default 10001 +# sys group daos_server +name: daos_server +# management service replicas +mgmt_svc_replicas: ['example1', 'example2', 'example3'] +# control listen port, default 10001 +# port: 10001 provider: ofi+tcp control_log_mask: DEBUG control_log_file: /tmp/daos_server.log diff --git a/utils/config/examples/daos_server_ucx.yml b/utils/config/examples/daos_server_ucx.yml index bdd35a4c647..bd413af495e 100644 --- a/utils/config/examples/daos_server_ucx.yml +++ b/utils/config/examples/daos_server_ucx.yml @@ -1,8 +1,11 @@ # Example configuration file for UCX -name: daos_server # sys group daos_server -access_points: ['example'] # management service leader (bootstrap) -# port: 10001 # control listen port, default 10001 +# sys group daos_server +name: daos_server +# management service replicas +mgmt_svc_replicas: ['example1', 'example2', 'example3'] +# control listen port, default 10001 +# port: 10001 # UCX providers: # diff --git a/utils/config/examples/daos_server_verbs.yml b/utils/config/examples/daos_server_verbs.yml index 667992351fc..32146674739 100644 --- a/utils/config/examples/daos_server_verbs.yml +++ b/utils/config/examples/daos_server_verbs.yml @@ -1,8 +1,11 @@ # Example configuration file for verbs -name: daos_server # sys group daos_server -access_points: ['example'] # management service leader (bootstrap) -# port: 10001 # control listen port, default 10001 +# sys group daos_server +name: daos_server +# management service replicas +mgmt_svc_replicas: ['example1', 'example2', 'example3'] +# control listen port, default 10001 +# port: 10001 provider: ofi+verbs control_log_mask: INFO control_log_file: /tmp/daos_server.log diff --git a/utils/docker/examples/daos-server/el8/daos_server.yml.example b/utils/docker/examples/daos-server/el8/daos_server.yml.example index 64d3baf7e79..8d0e90f8f07 100644 --- a/utils/docker/examples/daos-server/el8/daos_server.yml.example +++ b/utils/docker/examples/daos-server/el8/daos_server.yml.example @@ -6,7 +6,7 @@ # https://github.com/daos-stack/daos/blob/master/utils/config/daos_server.yml name: daos_server -access_points: ['localhost'] +mgmt_svc_replicas: ['localhost'] port: 10001 provider: ofi+tcp;ofi_rxm diff --git a/utils/docker/vcluster/daos-server/el8/daos_server.yml.in b/utils/docker/vcluster/daos-server/el8/daos_server.yml.in index 0d9ea2086f9..a0b1bd3ef01 100644 --- a/utils/docker/vcluster/daos-server/el8/daos_server.yml.in +++ b/utils/docker/vcluster/daos-server/el8/daos_server.yml.in @@ -1,7 +1,7 @@ # DAOS Server configuration file name: daos_server -access_points: ['daos-server'] +mgmt_svc_replicas: ['daos-server'] port: 10001 provider: ofi+tcp diff --git a/utils/nlt_server.yaml b/utils/nlt_server.yaml index d30dd9721bf..1e24e4d9c3c 100644 --- a/utils/nlt_server.yaml +++ b/utils/nlt_server.yaml @@ -3,7 +3,7 @@ port: 10001 provider: ofi+tcp disable_hugepages: true control_log_mask: DEBUG -access_points: ['localhost:10001'] +mgmt_svc_replicas: ['localhost:10001'] engines: - targets: 4 diff --git a/utils/node_local_test.py b/utils/node_local_test.py index 4d58999e5c1..727f2214ecf 100755 --- a/utils/node_local_test.py +++ b/utils/node_local_test.py @@ -779,7 +779,7 @@ def _start(self): agent_config = join(self.agent_dir, 'nlt_agent.yaml') with open(agent_config, 'w') as fd: agent_data = { - 'access_points': scyaml['access_points'], + 'access_points': scyaml['mgmt_svc_replicas'], 'control_log_mask': 'NOTICE', # INFO logs every client process connection } json.dump(agent_data, fd) From f07e5da4f984eacc6e68f8f735cdfd6d51e5993b Mon Sep 17 00:00:00 2001 From: Wang Shilong Date: Wed, 18 Dec 2024 11:05:35 +0800 Subject: [PATCH 10/10] DAOS-16867 object: fix error handling of ds_cpd_handle_one() (#15565) ABT_eventual_wait() might access freed eventual in error case. Signed-off-by: Wang Shilong --- src/object/srv_obj.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/object/srv_obj.c b/src/object/srv_obj.c index a79cec03f6f..8979b154b0c 100644 --- a/src/object/srv_obj.c +++ b/src/object/srv_obj.c @@ -4844,14 +4844,14 @@ ds_cpd_handle_one(crt_rpc_t *rpc, struct daos_cpd_sub_head *dcsh, struct daos_cp out: if (rc != 0) { if (bulks != NULL) { - for (i = 0; - i < dcde->dcde_write_cnt && rma_idx < rma; i++) { + for (i = 0; i < dcde->dcde_write_cnt; i++) { if (!bulks[i].inited) continue; - ABT_eventual_wait(bulks[i].eventual, NULL); - ABT_eventual_free(&bulks[i].eventual); - rma_idx++; + if (bulks[i].eventual != ABT_EVENTUAL_NULL) { + ABT_eventual_wait(bulks[i].eventual, NULL); + ABT_eventual_free(&bulks[i].eventual); + } } }