Skip to content

Commit

Permalink
Merge branch 'master' into benc-3515-remove-push-pull
Browse files Browse the repository at this point in the history
  • Loading branch information
benclifford authored Nov 14, 2024
2 parents f5104e5 + 92ab47f commit 7cec245
Show file tree
Hide file tree
Showing 5 changed files with 20 additions and 10 deletions.
13 changes: 9 additions & 4 deletions docs/quickstart.rst
Original file line number Diff line number Diff line change
Expand Up @@ -176,12 +176,17 @@ This script runs on a system that must stay on-line until all of your tasks comp
much computing power, such as the login node for a supercomputer.

The :class:`~parsl.config.Config` object holds definitions of Executors and the Providers and Launchers they rely on.
An example which launches 512 workers on 128 nodes of the Polaris supercomputer looks like
An example which launches 4 workers on 1 node of the Polaris supercomputer looks like

.. code-block:: python
from parsl import Config
from parsl.executors import HighThroughputExecutor
from parsl.providers import PBSProProvider
from parsl.launchers import MpiExecLauncher
config = Config(
retires=1, # Restart task if they fail once
retries=1, # Restart task if they fail once
executors=[
HighThroughputExecutor(
available_accelerators=4, # Maps one worker per GPU
Expand All @@ -191,13 +196,13 @@ An example which launches 512 workers on 128 nodes of the Polaris supercomputer
account="example",
worker_init="module load conda; conda activate parsl",
walltime="1:00:00",
queue="prod",
queue="debug",
scheduler_options="#PBS -l filesystems=home:eagle", # Change if data on other filesystem
launcher=MpiExecLauncher(
bind_cmd="--cpu-bind", overrides="--depth=64 --ppn 1"
), # Ensures 1 manger per node and allows it to divide work to all 64 cores
select_options="ngpus=4",
nodes_per_block=128,
nodes_per_block=1,
cpus_per_node=64,
),
),
Expand Down
6 changes: 6 additions & 0 deletions parsl/executors/taskvine/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,17 @@ def _set_manager_attributes(m, config):
# Enable peer transfer feature between workers if specified
if config.enable_peer_transfers:
m.enable_peer_transfers()
else:
m.disable_peer_transfers()

# Set catalog report to parsl if project name exists
if m.name:
m.set_property("framework", "parsl")

if config.tune_parameters is not None:
for k, v in config.tune_parameters.items():
m.tune(k, v)


def _prepare_environment_serverless(manager_config, env_cache_dir, poncho_create_script):
# Return path to a packaged poncho environment
Expand Down
5 changes: 5 additions & 0 deletions parsl/executors/taskvine/manager_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,10 @@ class TaskVineManagerConfig:
Directory to store TaskVine logging facilities.
Default is None, in which all TaskVine logs will be contained
in the Parsl logging directory.
tune_parameters: Optional[dict]
Extended vine_tune parameters, expressed in a dictionary
by { 'tune-parameter' : value }.
"""

# Connection and communication settings
Expand All @@ -181,6 +185,7 @@ class TaskVineManagerConfig:
autocategory: bool = True
enable_peer_transfers: bool = True
wait_for_workers: Optional[int] = None
tune_parameters: Optional[dict] = None

# Logging settings
vine_log_dir: Optional[str] = None
4 changes: 0 additions & 4 deletions parsl/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,10 +143,6 @@ def pytest_configure(config):
'markers',
'staging_required: Marks tests that require a staging provider, when there is no sharedFS'
)
config.addinivalue_line(
'markers',
'sshd_required: Marks tests that require a SSHD'
)
config.addinivalue_line(
'markers',
'multiple_cores_required: Marks tests that require multiple cores, such as htex affinity'
Expand Down
2 changes: 0 additions & 2 deletions test-requirements.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
flake8==6.1.0
pandas
paramiko
pytest>=7.4.0,<8
pytest-cov
pytest-random-order
Expand All @@ -10,7 +9,6 @@ mypy==1.5.1
types-mock
types-python-dateutil
types-requests
types-paramiko
mpi4py

# sqlalchemy is needed for typechecking, so it's here
Expand Down

0 comments on commit 7cec245

Please sign in to comment.