Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DAOS-15960 test: Makito's version of io_sys_admin test improvement #14858

Draft
wants to merge 25 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
73f7a8a
DAOS-15960 tests: Makito's version of io_sys_admin test improvement
Aug 1, 2024
963dc9a
DAOS-15960 test: Add launch.sh
Aug 1, 2024
b1795a7
DAOS-15960 test: Fix pylint
Aug 2, 2024
aca2967
DAOS-15960 test: Add comment
Aug 2, 2024
72d19ff
DAOS-15960 test: Remove -D 300
Aug 7, 2024
5c5e512
Merge branch 'master' into makito/DAOS-15960
Aug 7, 2024
8765979
Merge branch 'master' into makito/DAOS-15960
Aug 8, 2024
b410289
Merge branch 'master' into makito/DAOS-15960
Aug 9, 2024
77a0e43
Merge branch 'master' into makito/DAOS-15960
Aug 13, 2024
9aca1a2
DAOS-15960 test: Remove launch.sh because it's managed in aurora-tools
Aug 13, 2024
f1fe9c4
DAOS-15960 test: Remove unnecessary lines in io_sys_admin.yaml
Aug 13, 2024
05ff7de
DAOS-15960 test: Add line
Aug 13, 2024
5715158
Merge branch 'master' into makito/DAOS-15960
Aug 25, 2024
de365a5
DAOS-15960 test: Set oclass and cont_props to env var for HDF5-VOL
Aug 26, 2024
41cf512
Merge branch 'master' into makito/DAOS-15960
Aug 26, 2024
016628a
DAOS-15960 test: Remove unnecessary lines
Aug 26, 2024
4fa81b5
DAOS-15960 test: Get properties from self.container and modify it
Aug 28, 2024
660631a
Merge branch 'master' into makito/DAOS-15960
Aug 28, 2024
34b414c
Merge branch 'master' into makito/DAOS-15960
Sep 3, 2024
cc51aa2
DAOS-15960 test: Comment out unnecessary steps after file count test.
Sep 3, 2024
01d2ddb
DAOS-15960 test: Comment each line
Sep 3, 2024
4405ecf
Merge branch 'master' into makito/DAOS-15960
Sep 8, 2024
30955f2
Merge branch 'master' into makito/DAOS-15960
Sep 16, 2024
8b40e46
Merge branch 'master' into makito/DAOS-15960
Sep 21, 2024
90575bc
DAOS-15960 test: Set False to all subtests except io
Sep 21, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 19 additions & 16 deletions src/tests/ftest/deployment/io_sys_admin.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ container_3:
type: POSIX
properties: cksum:crc16,cksum_size:16384,srv_cksum:on
control_method: daos
oclass: EC_2P2GX
oclass: EC_16P2GX

pool:
size: 40%
Expand All @@ -60,35 +60,38 @@ container_set_owner:

largefilecount:
api:
- MPIIO
- POSIX
- HDF5
# We want to isolate the issues in HDF5 and HDF5-VOL, so comment out MPIIO, POSIX, and DFS.
#- MPIIO
#- POSIX
#- HDF5
- HDF5-VOL
- DFS
#- DFS
object_class:
ior_oclass: # Run once with SX and then with EC_16P2GX
- SX
- EC_2P1GX
- EC_16P2GX # Makito updated
mdtest_oclass: # Run once with S1 and then with EC_16P2G1
- S1
- EC_2P1G1
- EC_16P2G1 # Makito updated
ior:
client_processes:
ppn: 30
ppn: 104
dfs_destroy: False
flags: "-v -D 300 -W -w -k"
# 8/7: Removed -D 300
flags: -v -W -w -k
test_file: daos:testFile
repetitions: 1
signature: 123
transfer_size: '1Mib'
block_size: '1Mib'
# 8/1: 2Gib worked.
block_size: '2Gib'
dfuse:
disable_caching: True

mdtest:
client_processes:
ppn: 30
num_of_files_dirs: 100 # creating total of 1M files
ppn: 104
num_of_files_dirs: 1000 # creating total of 1M files
test_dir: "/"
iteration: 1
dfs_destroy: False
Expand All @@ -107,8 +110,8 @@ hdf5_vol:

io_sys_admin:
steps_to_run:
pool_create_ownership: True
storage_system_query: True
pool_create_ownership: False
storage_system_query: False
io: True
snapshot: True
datamover: True
snapshot: False
datamover: False
1 change: 1 addition & 0 deletions src/tests/ftest/util/file_count_test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@
intercept = os.path.join(self.prefix, 'lib64', 'libpil4dfs.so')
ior_oclass = self.params.get("ior_oclass", '/run/largefilecount/object_class/*')
mdtest_oclass = self.params.get("mdtest_oclass", '/run/largefilecount/object_class/*')
#cont_props = self.params.get("properties", '/run/container/*')

Check failure on line 79 in src/tests/ftest/util/file_count_test_base.py

View workflow job for this annotation

GitHub Actions / Flake8 check

E265 block comment should start with '# '

# create pool
self.add_pool(connect=False)
Expand Down
14 changes: 11 additions & 3 deletions src/tests/ftest/util/mdtest_test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,15 @@ def get_mdtest_container(self, pool):
params['dir_oclass'] = self.mdtest_cmd.dfs_dir_oclass.value
return self.get_container(pool, **params)

def execute_mdtest(self, out_queue=None, display_space=True, job_manager=None):
def execute_mdtest(self, out_queue=None, display_space=True, job_manager=None,
intercept=None):
"""Runner method for Mdtest.

Args:
out_queue (queue, optional): Pass any exceptions in a queue. Defaults to None.
display_space (bool, optional): Whether to display the pool space. Defaults to True.
job_manager (JobManager, optional): job manager used to run mdtest. Defaults to None.
intercept (default None): path to intercept library

Returns:
object: result of job manager run
Expand All @@ -102,7 +104,8 @@ def execute_mdtest(self, out_queue=None, display_space=True, job_manager=None):

# Run Mdtest
out = self.run_mdtest(
job_manager, self.processes, display_space=display_space, out_queue=out_queue)
job_manager, self.processes, display_space=display_space, out_queue=out_queue,
intercept=intercept)

if self.subprocess:
return out
Expand Down Expand Up @@ -134,7 +137,8 @@ def get_mdtest_job_manager_command(self, mpi_type):
manager = get_job_manager(self, "Orterun", self.mdtest_cmd, self.subprocess)
return manager

def run_mdtest(self, manager, processes, display_space=True, pool=None, out_queue=None):
def run_mdtest(self, manager, processes, display_space=True, pool=None, out_queue=None,
intercept=None):
"""Run the Mdtest command.

Args:
Expand All @@ -150,6 +154,10 @@ def run_mdtest(self, manager, processes, display_space=True, pool=None, out_queu
env = self.mdtest_cmd.get_default_env(str(manager), self.client_log)
manager.assign_hosts(self.hostlist_clients, self.workdir, self.hostfile_clients_slots)
# Pass only processes or ppn to be compatible with previous behavior
if intercept:
env['LD_PRELOAD'] = intercept
if 'D_IL_REPORT' not in env:
env['D_IL_REPORT'] = '1'
if self.ppn is not None:
manager.assign_processes(ppn=self.ppn)
else:
Expand Down
Loading