-
Notifications
You must be signed in to change notification settings - Fork 23
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #128 from ReactionMechanismGenerator/local2
Run ARC on a server
- Loading branch information
Showing
18 changed files
with
688 additions
and
346 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -3,5 +3,6 @@ | |
|
|
||
| import job | ||
| import ssh | ||
| import local | ||
| import inputs | ||
| import submit | ||
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| #!/usr/bin/env python | ||
| # encoding: utf-8 | ||
|
|
||
| """ | ||
| A module for running jobs on the local machine | ||
| When transitioning to Python 3, use subprocess.run(), https://docs.python.org/3/library/subprocess.html#subprocess.run | ||
| """ | ||
|
|
||
| from __future__ import (absolute_import, division, print_function, unicode_literals) | ||
| import subprocess | ||
| import os | ||
| import shutil | ||
| import datetime | ||
|
|
||
| from arc.job.ssh import check_job_status_in_stdout | ||
| from arc.settings import servers, check_status_command, submit_command, submit_filename, delete_command, output_filename | ||
|
|
||
| ################################################################## | ||
|
|
||
|
|
||
| def execute_command(command, shell=True): | ||
| """ | ||
| Execute a command. `command` is an array of string commands to send. | ||
| If path is not an empty string, the command will be executed in the directory path it points to. | ||
| Returns lists of stdin, stdout, stderr corresponding to the commands sent. | ||
| `shell` specifies whether the command should be executed using bash instead of Python | ||
| """ | ||
| if not isinstance(command, list) and not shell: | ||
| command = [command] | ||
| stdout = subprocess.check_output(command, shell=shell) | ||
| return stdout.splitlines(), '' | ||
|
|
||
|
|
||
| def check_job_status(job_id): | ||
| """ | ||
| Possible statuses: `before_submission`, `running`, `errored on node xx`, `done` | ||
| Status line formats: | ||
| pharos: '540420 0.45326 xq1340b user_name r 10/26/2018 11:08:30 [email protected]' | ||
| rmg: '14428 debug xq1371m2 user_name R 50-04:04:46 1 node06' | ||
| """ | ||
| server = 'local' | ||
| cmd = check_status_command[servers[server]['cluster_soft']] + ' -u ' + servers[server]['un'] | ||
| stdout = execute_command(cmd)[0] | ||
| return check_job_status_in_stdout(job_id=job_id, stdout=stdout, server=server) | ||
|
|
||
|
|
||
| def delete_job(job_id): | ||
| """ | ||
| Deletes a running job | ||
| """ | ||
| cmd = delete_command[servers['local']['cluster_soft']] + ' ' + str(job_id) | ||
| execute_command(cmd) | ||
|
|
||
|
|
||
| def check_running_jobs_ids(): | ||
| """ | ||
| Return a list of ``int`` representing job IDs of all jobs submitted by the user on a server | ||
| """ | ||
| running_jobs_ids = list() | ||
| cmd = check_status_command[servers['local']['cluster_soft']] + ' -u ' + servers['local']['un'] | ||
| stdout = execute_command(cmd)[0] | ||
| for i, status_line in enumerate(stdout): | ||
| if (servers['local']['cluster_soft'].lower() == 'slurm' and i > 0)\ | ||
| or (servers['local']['cluster_soft'].lower() == 'oge' and i > 1): | ||
| running_jobs_ids.append(int(status_line.split()[0])) | ||
| return running_jobs_ids | ||
|
|
||
|
|
||
| def submit_job(path): | ||
| """ | ||
| Submit a job | ||
| `path` is the job's folder path, where the submit script is located (without the submit script file name) | ||
| """ | ||
| job_status = '' | ||
| job_id = 0 | ||
| cmd = 'cd ' + path + '; ' + submit_command[servers['local']['cluster_soft']] + ' '\ | ||
| + submit_filename[servers['local']['cluster_soft']] | ||
| stdout = execute_command(cmd)[0] | ||
| if 'submitted' in stdout[0].lower(): | ||
| job_status = 'running' | ||
| if servers['local']['cluster_soft'].lower() == 'oge': | ||
| job_id = int(stdout[0].split()[2]) | ||
| elif servers['local']['cluster_soft'].lower() == 'slurm': | ||
| job_id = int(stdout[0].split()[3]) | ||
| else: | ||
| raise ValueError('Unrecognized cluster software {0}'.format(servers['local']['cluster_soft'])) | ||
| return job_status, job_id | ||
|
|
||
|
|
||
| def get_last_modified_time(file_path): | ||
| """returns the last modified time of `file_path` in a datetime format""" | ||
| try: | ||
| timestamp = os.stat(file_path).st_mtime | ||
| except (IOError, OSError): | ||
| return None | ||
| return datetime.datetime.fromtimestamp(timestamp) | ||
|
|
||
|
|
||
| def write_file(file_path, file_string): | ||
| """ | ||
| Write `file_string` as the file's content in `file_path` | ||
| """ | ||
| with open(file_path, 'w') as f: | ||
| f.write(file_string) | ||
|
|
||
|
|
||
| def rename_output(local_file_path, software): | ||
| """ | ||
| Rename the output file to "output.out" for consistency between software | ||
| `local_file_path` is the full path to the output.out file, | ||
| `software` is the software used for the job by which the original output file name is determined | ||
| """ | ||
| software = software.lower() | ||
| if os.path.isfile(os.path.join(os.path.dirname(local_file_path), output_filename[software])): | ||
| shutil.move(src=os.path.join(os.path.dirname(local_file_path), output_filename[software]), dst=local_file_path) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| #!/usr/bin/env python | ||
| # -*- coding: utf-8 -*- | ||
|
|
||
| """ | ||
| This module contains unit tests of the arc.job.local module | ||
| """ | ||
|
|
||
| from __future__ import (absolute_import, division, print_function, unicode_literals) | ||
| import unittest | ||
| import os | ||
| import shutil | ||
| import datetime | ||
|
|
||
| import arc.job.local as local | ||
| from arc.settings import arc_path | ||
|
|
||
| ################################################################################ | ||
|
|
||
|
|
||
| class TestLocal(unittest.TestCase): | ||
| """ | ||
| Contains unit tests for the local module | ||
| """ | ||
|
|
||
| def test_execute_command(self): | ||
| """Test executing a local command""" | ||
| command1 = 'ls' | ||
| out1 = local.execute_command(command1) | ||
| self.assertIsInstance(out1, tuple) | ||
| self.assertIsInstance(out1[0], list) | ||
| self.assertEqual(out1[1], '') | ||
| self.assertIn('arc', out1[0]) | ||
| self.assertIn('ARC.py', out1[0]) | ||
| self.assertIn('environment.yml', out1[0]) | ||
|
|
||
| def test_get_last_modified_time(self): | ||
| """Test the get_last_modified_time() function""" | ||
| path = os.path.join(arc_path, 'ARC.py') | ||
| t = local.get_last_modified_time(path) | ||
| self.assertIsInstance(t, datetime.datetime) | ||
|
|
||
| def test_rename_output(self): | ||
| """Test the rename_output() function""" | ||
| path1 = os.path.join(arc_path, 'scratch', 'input.log') | ||
| path2 = os.path.join(arc_path, 'scratch', 'output.out') | ||
| if not os.path.exists(os.path.join(arc_path, 'scratch')): | ||
| os.makedirs(os.path.join(arc_path, 'scratch')) | ||
| with open(path1, 'w'): | ||
| pass | ||
| local.rename_output(local_file_path=path2, software='gaussian') | ||
| self.assertFalse(os.path.isfile(path1)) | ||
| self.assertTrue(os.path.isfile(path2)) | ||
| shutil.rmtree(os.path.join(arc_path, 'scratch')) | ||
|
|
||
| ################################################################################ | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| unittest.main(testRunner=unittest.TextTestRunner(verbosity=2)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| #!/usr/bin/env python | ||
| # -*- coding: utf-8 -*- | ||
|
|
||
| """ | ||
| This module contains unit tests of the arc.job.ssh module | ||
| """ | ||
|
|
||
| from __future__ import (absolute_import, division, print_function, unicode_literals) | ||
| import unittest | ||
|
|
||
| import arc.job.ssh as ssh | ||
|
|
||
| ################################################################################ | ||
|
|
||
|
|
||
| class TestSSH(unittest.TestCase): | ||
| """ | ||
| Contains unit tests for the SSH module | ||
| """ | ||
|
|
||
| def test_check_job_status_in_stdout(self): | ||
| """Test checking the job status in stdout""" | ||
| stdout = """job-ID prior name user state submit/start at queue slots ja-task-ID | ||
| ----------------------------------------------------------------------------------------------------------------- | ||
| 582682 0.45451 a9654 alongd e 04/17/2019 16:22:14 [email protected] 48 | ||
| 588334 0.45451 pf1005a alongd r 05/07/2019 16:24:31 [email protected] 48 | ||
| 588345 0.45451 a14121 alongd r 05/08/2019 02:11:42 [email protected] 48 """ | ||
| status1 = ssh.check_job_status_in_stdout(job_id=588345, stdout=stdout, server='server1') | ||
| self.assertEqual(status1, 'running') | ||
| status2 = ssh.check_job_status_in_stdout(job_id=582682, stdout=stdout, server='server1') | ||
| self.assertEqual(status2, 'errored') | ||
| status3 = ssh.check_job_status_in_stdout(job_id=582600, stdout=stdout, server='server1') | ||
| self.assertEqual(status3, 'done') | ||
|
|
||
| ################################################################################ | ||
|
|
||
|
|
||
| if __name__ == '__main__': | ||
| unittest.main(testRunner=unittest.TextTestRunner(verbosity=2)) |
Oops, something went wrong.