From fc4972e33e1acdc79ba311cbe7843bee5037b7ad Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 21 Feb 2024 10:56:45 -0700 Subject: [PATCH 01/28] Update to what we expect cesm2_3_beta17 externals will be, fixing #2351 --- Externals.cfg | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index a17f8e2ec6..21499fc452 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -8,7 +8,7 @@ required = True local_path = components/cism protocol = git repo_url = https://github.com/ESCOMP/CISM-wrapper -tag = cismwrap_2_1_96 +tag = cismwrap_2_1_97 externals = Externals_CISM.cfg required = True @@ -16,14 +16,14 @@ required = True local_path = components/rtm protocol = git repo_url = https://github.com/ESCOMP/RTM -tag = rtm1_0_78 +tag = rtm1_0_79 required = True [mosart] local_path = components/mosart protocol = git repo_url = https://github.com/ESCOMP/MOSART -tag = mosart1_0_48 +tag = mosart1_0_49 required = True [mizuRoute] @@ -34,7 +34,7 @@ hash = 34723c2 required = True [ccs_config] -tag = ccs_config_cesm0.0.84 +tag = ccs_config_cesm0.0.92 protocol = git repo_url = https://github.com/ESMCI/ccs_config_cesm.git local_path = ccs_config @@ -44,18 +44,18 @@ required = True local_path = cime protocol = git repo_url = https://github.com/ESMCI/cime -tag = cime6.0.175 +tag = cime6.0.217_httpsbranch01 required = True [cmeps] -tag = cmeps0.14.43 +tag = cmeps0.14.50 protocol = git repo_url = https://github.com/ESCOMP/CMEPS.git local_path = components/cmeps required = True [cdeps] -tag = cdeps1.0.24 +tag = cdeps1.0.28 protocol = git repo_url = https://github.com/ESCOMP/CDEPS.git local_path = components/cdeps @@ -70,7 +70,7 @@ local_path = components/cpl7 required = True [share] -tag = share1.0.17 +tag = share1.0.18 protocol = git repo_url = https://github.com/ESCOMP/CESM_share local_path = share From 44929cb871313730678ab52a3fd6a7e22d25dc5b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 21 Feb 2024 10:58:37 -0700 Subject: [PATCH 02/28] Squashed 'manage_externals/' changes from 7b6d92ef6..0f884bfec 0f884bfec Merge pull request #205 from jedwards4b/sunset_svn_git_access 82a5edf79 merge in billsacks:svn_testing_no_github 17532c160 Use a local svn repo for testing 9c904341a different method to determine if in tests 539952ebd remove debug print statement cc5434fa7 fix submodule testing 1d7f28840 remove broken tests 04e94a519 provide a meaningful error message 38bcc0a8c Merge pull request #201 from jedwards4b/partial_match b4466a5aa remove debug print statement c3cf3ec35 fix issue with partial branch match git-subtree-dir: manage_externals git-subtree-split: 0f884bfec8e43d0c02261de858d6ec3f6d855e51 --- manic/repository_git.py | 18 +- manic/repository_svn.py | 3 + test/repos/README.md | 4 +- test/repos/simple-ext.svn/README.txt | 5 + test/repos/simple-ext.svn/conf/authz | 32 +++ test/repos/simple-ext.svn/conf/hooks-env.tmpl | 19 ++ test/repos/simple-ext.svn/conf/passwd | 8 + test/repos/simple-ext.svn/conf/svnserve.conf | 81 +++++++ test/repos/simple-ext.svn/db/current | 1 + test/repos/simple-ext.svn/db/format | 3 + test/repos/simple-ext.svn/db/fs-type | 1 + test/repos/simple-ext.svn/db/fsfs.conf | 200 ++++++++++++++++++ test/repos/simple-ext.svn/db/min-unpacked-rev | 1 + test/repos/simple-ext.svn/db/rep-cache.db | Bin 0 -> 8192 bytes .../simple-ext.svn/db/rep-cache.db-journal | 0 test/repos/simple-ext.svn/db/revprops/0/0 | 5 + test/repos/simple-ext.svn/db/revprops/0/1 | 13 ++ test/repos/simple-ext.svn/db/revprops/0/2 | 13 ++ test/repos/simple-ext.svn/db/revprops/0/3 | 13 ++ test/repos/simple-ext.svn/db/revs/0/0 | Bin 0 -> 253 bytes test/repos/simple-ext.svn/db/revs/0/1 | Bin 0 -> 725 bytes test/repos/simple-ext.svn/db/revs/0/2 | Bin 0 -> 816 bytes test/repos/simple-ext.svn/db/revs/0/3 | Bin 0 -> 769 bytes test/repos/simple-ext.svn/db/txn-current | 1 + test/repos/simple-ext.svn/db/txn-current-lock | 0 test/repos/simple-ext.svn/db/uuid | 2 + test/repos/simple-ext.svn/db/write-lock | 0 test/repos/simple-ext.svn/format | 1 + .../simple-ext.svn/hooks/post-commit.tmpl | 62 ++++++ .../repos/simple-ext.svn/hooks/post-lock.tmpl | 64 ++++++ .../hooks/post-revprop-change.tmpl | 69 ++++++ .../simple-ext.svn/hooks/post-unlock.tmpl | 61 ++++++ .../simple-ext.svn/hooks/pre-commit.tmpl | 91 ++++++++ test/repos/simple-ext.svn/hooks/pre-lock.tmpl | 95 +++++++++ .../hooks/pre-revprop-change.tmpl | 79 +++++++ .../simple-ext.svn/hooks/pre-unlock.tmpl | 87 ++++++++ .../simple-ext.svn/hooks/start-commit.tmpl | 81 +++++++ test/repos/simple-ext.svn/locks/db-logs.lock | 3 + test/repos/simple-ext.svn/locks/db.lock | 3 + test/test_sys_checkout.py | 51 ++--- 40 files changed, 1126 insertions(+), 44 deletions(-) create mode 100644 test/repos/simple-ext.svn/README.txt create mode 100644 test/repos/simple-ext.svn/conf/authz create mode 100644 test/repos/simple-ext.svn/conf/hooks-env.tmpl create mode 100644 test/repos/simple-ext.svn/conf/passwd create mode 100644 test/repos/simple-ext.svn/conf/svnserve.conf create mode 100644 test/repos/simple-ext.svn/db/current create mode 100644 test/repos/simple-ext.svn/db/format create mode 100644 test/repos/simple-ext.svn/db/fs-type create mode 100644 test/repos/simple-ext.svn/db/fsfs.conf create mode 100644 test/repos/simple-ext.svn/db/min-unpacked-rev create mode 100644 test/repos/simple-ext.svn/db/rep-cache.db create mode 100644 test/repos/simple-ext.svn/db/rep-cache.db-journal create mode 100644 test/repos/simple-ext.svn/db/revprops/0/0 create mode 100644 test/repos/simple-ext.svn/db/revprops/0/1 create mode 100644 test/repos/simple-ext.svn/db/revprops/0/2 create mode 100644 test/repos/simple-ext.svn/db/revprops/0/3 create mode 100644 test/repos/simple-ext.svn/db/revs/0/0 create mode 100644 test/repos/simple-ext.svn/db/revs/0/1 create mode 100644 test/repos/simple-ext.svn/db/revs/0/2 create mode 100644 test/repos/simple-ext.svn/db/revs/0/3 create mode 100644 test/repos/simple-ext.svn/db/txn-current create mode 100644 test/repos/simple-ext.svn/db/txn-current-lock create mode 100644 test/repos/simple-ext.svn/db/uuid create mode 100644 test/repos/simple-ext.svn/db/write-lock create mode 100644 test/repos/simple-ext.svn/format create mode 100755 test/repos/simple-ext.svn/hooks/post-commit.tmpl create mode 100755 test/repos/simple-ext.svn/hooks/post-lock.tmpl create mode 100755 test/repos/simple-ext.svn/hooks/post-revprop-change.tmpl create mode 100755 test/repos/simple-ext.svn/hooks/post-unlock.tmpl create mode 100755 test/repos/simple-ext.svn/hooks/pre-commit.tmpl create mode 100755 test/repos/simple-ext.svn/hooks/pre-lock.tmpl create mode 100755 test/repos/simple-ext.svn/hooks/pre-revprop-change.tmpl create mode 100755 test/repos/simple-ext.svn/hooks/pre-unlock.tmpl create mode 100755 test/repos/simple-ext.svn/hooks/start-commit.tmpl create mode 100644 test/repos/simple-ext.svn/locks/db-logs.lock create mode 100644 test/repos/simple-ext.svn/locks/db.lock mode change 100644 => 100755 test/test_sys_checkout.py diff --git a/manic/repository_git.py b/manic/repository_git.py index adc666cc57..aab1a468a8 100644 --- a/manic/repository_git.py +++ b/manic/repository_git.py @@ -7,6 +7,7 @@ import copy import os +import sys from .global_constants import EMPTY_STR, LOCAL_PATH_INDICATOR from .global_constants import VERBOSITY_VERBOSE @@ -380,7 +381,6 @@ def _check_for_valid_ref(self, ref, remote_name, dirname): is_tag = self._ref_is_tag(ref, dirname) is_branch = self._ref_is_branch(ref, remote_name, dirname) is_hash = self._ref_is_hash(ref, dirname) - is_valid = is_tag or is_branch or is_hash if not is_valid: msg = ('In repo "{0}": reference "{1}" does not appear to be a ' @@ -710,7 +710,10 @@ def _git_lsremote_branch(ref, remote_name, dirname): cmd = ('git -C {dirname} ls-remote --exit-code --heads ' '{remote_name} {ref}').format( dirname=dirname, remote_name=remote_name, ref=ref).split() - status = execute_subprocess(cmd, status_to_caller=True) + status, output = execute_subprocess(cmd, status_to_caller=True, output_to_caller=True) + if not status and not f"refs/heads/{ref}" in output: + # In this case the ref is contained in the branch name but is not the complete branch name + return -1 return status @staticmethod @@ -837,12 +840,19 @@ def _git_update_submodules(verbosity, dirname): """Run git submodule update for the side effect of updating this repo's submodules. """ + # due to https://vielmetti.typepad.com/logbook/2022/10/git-security-fixes-lead-to-fatal-transport-file-not-allowed-error-in-ci-systems-cve-2022-39253.html + # submodules from file doesn't work without overriding the protocol, this is done + # for testing submodule support but should not be done in practice + file_protocol = "" + if 'unittest' in sys.modules.keys(): + file_protocol = "-c protocol.file.allow=always" + # First, verify that we have a .gitmodules file if os.path.exists( os.path.join(dirname, ExternalsDescription.GIT_SUBMODULES_FILENAME)): - cmd = ('git -C {dirname} submodule update --init --recursive' - .format(dirname=dirname)).split() + cmd = ('git {file_protocol} -C {dirname} submodule update --init --recursive' + .format(file_protocol=file_protocol, dirname=dirname)).split() if verbosity >= VERBOSITY_VERBOSE: printlog(' {0}'.format(' '.join(cmd))) diff --git a/manic/repository_svn.py b/manic/repository_svn.py index 922855d34e..32a71184b4 100644 --- a/manic/repository_svn.py +++ b/manic/repository_svn.py @@ -42,6 +42,9 @@ def __init__(self, component_name, repo, ignore_ancestry=False): Parse repo (a XML element). """ Repository.__init__(self, component_name, repo) + if 'github.com' in self._url: + msg = "SVN access to github.com is no longer supported" + fatal_error(msg) self._ignore_ancestry = ignore_ancestry if self._url.endswith('/'): # there is already a '/' separator in the URL; no need to add another diff --git a/test/repos/README.md b/test/repos/README.md index 8a3502c35f..026b684ea3 100644 --- a/test/repos/README.md +++ b/test/repos/README.md @@ -1,6 +1,6 @@ -Git repositories for testing git-related behavior. For usage and terminology notes, see test/test_sys_checkout.py. +Git and svn repositories for testing git and svn-related behavior. For usage and terminology notes, see test/test_sys_checkout.py. -To list files and view file contents at HEAD: +For git repos: To list files and view file contents at HEAD: ``` cd git ls-tree --full-tree -r --name-only HEAD diff --git a/test/repos/simple-ext.svn/README.txt b/test/repos/simple-ext.svn/README.txt new file mode 100644 index 0000000000..9935818a1b --- /dev/null +++ b/test/repos/simple-ext.svn/README.txt @@ -0,0 +1,5 @@ +This is a Subversion repository; use the 'svnadmin' and 'svnlook' +tools to examine it. Do not add, delete, or modify files here +unless you know how to avoid corrupting the repository. + +Visit http://subversion.apache.org/ for more information. diff --git a/test/repos/simple-ext.svn/conf/authz b/test/repos/simple-ext.svn/conf/authz new file mode 100644 index 0000000000..0b9a41074e --- /dev/null +++ b/test/repos/simple-ext.svn/conf/authz @@ -0,0 +1,32 @@ +### This file is an example authorization file for svnserve. +### Its format is identical to that of mod_authz_svn authorization +### files. +### As shown below each section defines authorizations for the path and +### (optional) repository specified by the section name. +### The authorizations follow. An authorization line can refer to: +### - a single user, +### - a group of users defined in a special [groups] section, +### - an alias defined in a special [aliases] section, +### - all authenticated users, using the '$authenticated' token, +### - only anonymous users, using the '$anonymous' token, +### - anyone, using the '*' wildcard. +### +### A match can be inverted by prefixing the rule with '~'. Rules can +### grant read ('r') access, read-write ('rw') access, or no access +### (''). + +[aliases] +# joe = /C=XZ/ST=Dessert/L=Snake City/O=Snake Oil, Ltd./OU=Research Institute/CN=Joe Average + +[groups] +# harry_and_sally = harry,sally +# harry_sally_and_joe = harry,sally,&joe + +# [/foo/bar] +# harry = rw +# &joe = r +# * = + +# [repository:/baz/fuz] +# @harry_and_sally = rw +# * = r diff --git a/test/repos/simple-ext.svn/conf/hooks-env.tmpl b/test/repos/simple-ext.svn/conf/hooks-env.tmpl new file mode 100644 index 0000000000..ee965c316c --- /dev/null +++ b/test/repos/simple-ext.svn/conf/hooks-env.tmpl @@ -0,0 +1,19 @@ +### This file is an example hook script environment configuration file. +### Hook scripts run in an empty environment by default. +### As shown below each section defines environment variables for a +### particular hook script. The [default] section defines environment +### variables for all hook scripts, unless overridden by a hook-specific +### section. + +### This example configures a UTF-8 locale for all hook scripts, so that +### special characters, such as umlauts, may be printed to stderr. +### If UTF-8 is used with a mod_dav_svn server, the SVNUseUTF8 option must +### also be set to 'yes' in httpd.conf. +### With svnserve, the LANG environment variable of the svnserve process +### must be set to the same value as given here. +[default] +LANG = en_US.UTF-8 + +### This sets the PATH environment variable for the pre-commit hook. +[pre-commit] +PATH = /usr/local/bin:/usr/bin:/usr/sbin diff --git a/test/repos/simple-ext.svn/conf/passwd b/test/repos/simple-ext.svn/conf/passwd new file mode 100644 index 0000000000..ecaa08dcec --- /dev/null +++ b/test/repos/simple-ext.svn/conf/passwd @@ -0,0 +1,8 @@ +### This file is an example password file for svnserve. +### Its format is similar to that of svnserve.conf. As shown in the +### example below it contains one section labelled [users]. +### The name and password for each user follow, one account per line. + +[users] +# harry = harryssecret +# sally = sallyssecret diff --git a/test/repos/simple-ext.svn/conf/svnserve.conf b/test/repos/simple-ext.svn/conf/svnserve.conf new file mode 100644 index 0000000000..6cefc17b3e --- /dev/null +++ b/test/repos/simple-ext.svn/conf/svnserve.conf @@ -0,0 +1,81 @@ +### This file controls the configuration of the svnserve daemon, if you +### use it to allow access to this repository. (If you only allow +### access through http: and/or file: URLs, then this file is +### irrelevant.) + +### Visit http://subversion.apache.org/ for more information. + +[general] +### The anon-access and auth-access options control access to the +### repository for unauthenticated (a.k.a. anonymous) users and +### authenticated users, respectively. +### Valid values are "write", "read", and "none". +### Setting the value to "none" prohibits both reading and writing; +### "read" allows read-only access, and "write" allows complete +### read/write access to the repository. +### The sample settings below are the defaults and specify that anonymous +### users have read-only access to the repository, while authenticated +### users have read and write access to the repository. +# anon-access = read +# auth-access = write +### The password-db option controls the location of the password +### database file. Unless you specify a path starting with a /, +### the file's location is relative to the directory containing +### this configuration file. +### If SASL is enabled (see below), this file will NOT be used. +### Uncomment the line below to use the default password file. +# password-db = passwd +### The authz-db option controls the location of the authorization +### rules for path-based access control. Unless you specify a path +### starting with a /, the file's location is relative to the +### directory containing this file. The specified path may be a +### repository relative URL (^/) or an absolute file:// URL to a text +### file in a Subversion repository. If you don't specify an authz-db, +### no path-based access control is done. +### Uncomment the line below to use the default authorization file. +# authz-db = authz +### The groups-db option controls the location of the file with the +### group definitions and allows maintaining groups separately from the +### authorization rules. The groups-db file is of the same format as the +### authz-db file and should contain a single [groups] section with the +### group definitions. If the option is enabled, the authz-db file cannot +### contain a [groups] section. Unless you specify a path starting with +### a /, the file's location is relative to the directory containing this +### file. The specified path may be a repository relative URL (^/) or an +### absolute file:// URL to a text file in a Subversion repository. +### This option is not being used by default. +# groups-db = groups +### This option specifies the authentication realm of the repository. +### If two repositories have the same authentication realm, they should +### have the same password database, and vice versa. The default realm +### is repository's uuid. +# realm = My First Repository +### The force-username-case option causes svnserve to case-normalize +### usernames before comparing them against the authorization rules in the +### authz-db file configured above. Valid values are "upper" (to upper- +### case the usernames), "lower" (to lowercase the usernames), and +### "none" (to compare usernames as-is without case conversion, which +### is the default behavior). +# force-username-case = none +### The hooks-env options specifies a path to the hook script environment +### configuration file. This option overrides the per-repository default +### and can be used to configure the hook script environment for multiple +### repositories in a single file, if an absolute path is specified. +### Unless you specify an absolute path, the file's location is relative +### to the directory containing this file. +# hooks-env = hooks-env + +[sasl] +### This option specifies whether you want to use the Cyrus SASL +### library for authentication. Default is false. +### Enabling this option requires svnserve to have been built with Cyrus +### SASL support; to check, run 'svnserve --version' and look for a line +### reading 'Cyrus SASL authentication is available.' +# use-sasl = true +### These options specify the desired strength of the security layer +### that you want SASL to provide. 0 means no encryption, 1 means +### integrity-checking only, values larger than 1 are correlated +### to the effective key length for encryption (e.g. 128 means 128-bit +### encryption). The values below are the defaults. +# min-encryption = 0 +# max-encryption = 256 diff --git a/test/repos/simple-ext.svn/db/current b/test/repos/simple-ext.svn/db/current new file mode 100644 index 0000000000..00750edc07 --- /dev/null +++ b/test/repos/simple-ext.svn/db/current @@ -0,0 +1 @@ +3 diff --git a/test/repos/simple-ext.svn/db/format b/test/repos/simple-ext.svn/db/format new file mode 100644 index 0000000000..5dd0c22198 --- /dev/null +++ b/test/repos/simple-ext.svn/db/format @@ -0,0 +1,3 @@ +8 +layout sharded 1000 +addressing logical diff --git a/test/repos/simple-ext.svn/db/fs-type b/test/repos/simple-ext.svn/db/fs-type new file mode 100644 index 0000000000..4fdd95313f --- /dev/null +++ b/test/repos/simple-ext.svn/db/fs-type @@ -0,0 +1 @@ +fsfs diff --git a/test/repos/simple-ext.svn/db/fsfs.conf b/test/repos/simple-ext.svn/db/fsfs.conf new file mode 100644 index 0000000000..ac6877a727 --- /dev/null +++ b/test/repos/simple-ext.svn/db/fsfs.conf @@ -0,0 +1,200 @@ +### This file controls the configuration of the FSFS filesystem. + +[memcached-servers] +### These options name memcached servers used to cache internal FSFS +### data. See http://www.danga.com/memcached/ for more information on +### memcached. To use memcached with FSFS, run one or more memcached +### servers, and specify each of them as an option like so: +# first-server = 127.0.0.1:11211 +# remote-memcached = mymemcached.corp.example.com:11212 +### The option name is ignored; the value is of the form HOST:PORT. +### memcached servers can be shared between multiple repositories; +### however, if you do this, you *must* ensure that repositories have +### distinct UUIDs and paths, or else cached data from one repository +### might be used by another accidentally. Note also that memcached has +### no authentication for reads or writes, so you must ensure that your +### memcached servers are only accessible by trusted users. + +[caches] +### When a cache-related error occurs, normally Subversion ignores it +### and continues, logging an error if the server is appropriately +### configured (and ignoring it with file:// access). To make +### Subversion never ignore cache errors, uncomment this line. +# fail-stop = true + +[rep-sharing] +### To conserve space, the filesystem can optionally avoid storing +### duplicate representations. This comes at a slight cost in +### performance, as maintaining a database of shared representations can +### increase commit times. The space savings are dependent upon the size +### of the repository, the number of objects it contains and the amount of +### duplication between them, usually a function of the branching and +### merging process. +### +### The following parameter enables rep-sharing in the repository. It can +### be switched on and off at will, but for best space-saving results +### should be enabled consistently over the life of the repository. +### 'svnadmin verify' will check the rep-cache regardless of this setting. +### rep-sharing is enabled by default. +# enable-rep-sharing = true + +[deltification] +### To conserve space, the filesystem stores data as differences against +### existing representations. This comes at a slight cost in performance, +### as calculating differences can increase commit times. Reading data +### will also create higher CPU load and the data will be fragmented. +### Since deltification tends to save significant amounts of disk space, +### the overall I/O load can actually be lower. +### +### The options in this section allow for tuning the deltification +### strategy. Their effects on data size and server performance may vary +### from one repository to another. Versions prior to 1.8 will ignore +### this section. +### +### The following parameter enables deltification for directories. It can +### be switched on and off at will, but for best space-saving results +### should be enabled consistently over the lifetime of the repository. +### Repositories containing large directories will benefit greatly. +### In rarely accessed repositories, the I/O overhead may be significant +### as caches will most likely be low. +### directory deltification is enabled by default. +# enable-dir-deltification = true +### +### The following parameter enables deltification for properties on files +### and directories. Overall, this is a minor tuning option but can save +### some disk space if you merge frequently or frequently change node +### properties. You should not activate this if rep-sharing has been +### disabled because this may result in a net increase in repository size. +### property deltification is enabled by default. +# enable-props-deltification = true +### +### During commit, the server may need to walk the whole change history of +### of a given node to find a suitable deltification base. This linear +### process can impact commit times, svnadmin load and similar operations. +### This setting limits the depth of the deltification history. If the +### threshold has been reached, the node will be stored as fulltext and a +### new deltification history begins. +### Note, this is unrelated to svn log. +### Very large values rarely provide significant additional savings but +### can impact performance greatly - in particular if directory +### deltification has been activated. Very small values may be useful in +### repositories that are dominated by large, changing binaries. +### Should be a power of two minus 1. A value of 0 will effectively +### disable deltification. +### For 1.8, the default value is 1023; earlier versions have no limit. +# max-deltification-walk = 1023 +### +### The skip-delta scheme used by FSFS tends to repeatably store redundant +### delta information where a simple delta against the latest version is +### often smaller. By default, 1.8+ will therefore use skip deltas only +### after the linear chain of deltas has grown beyond the threshold +### specified by this setting. +### Values up to 64 can result in some reduction in repository size for +### the cost of quickly increasing I/O and CPU costs. Similarly, smaller +### numbers can reduce those costs at the cost of more disk space. For +### rarely read repositories or those containing larger binaries, this may +### present a better trade-off. +### Should be a power of two. A value of 1 or smaller will cause the +### exclusive use of skip-deltas (as in pre-1.8). +### For 1.8, the default value is 16; earlier versions use 1. +# max-linear-deltification = 16 +### +### After deltification, we compress the data to minimize on-disk size. +### This setting controls the compression algorithm, which will be used in +### future revisions. It can be used to either disable compression or to +### select between available algorithms (zlib, lz4). zlib is a general- +### purpose compression algorithm. lz4 is a fast compression algorithm +### which should be preferred for repositories with large and, possibly, +### incompressible files. Note that the compression ratio of lz4 is +### usually lower than the one provided by zlib, but using it can +### significantly speed up commits as well as reading the data. +### lz4 compression algorithm is supported, starting from format 8 +### repositories, available in Subversion 1.10 and higher. +### The syntax of this option is: +### compression = none | lz4 | zlib | zlib-1 ... zlib-9 +### Versions prior to Subversion 1.10 will ignore this option. +### The default value is 'lz4' if supported by the repository format and +### 'zlib' otherwise. 'zlib' is currently equivalent to 'zlib-5'. +# compression = lz4 +### +### DEPRECATED: The new 'compression' option deprecates previously used +### 'compression-level' option, which was used to configure zlib compression. +### For compatibility with previous versions of Subversion, this option can +### still be used (and it will result in zlib compression with the +### corresponding compression level). +### compression-level = 0 ... 9 (default is 5) + +[packed-revprops] +### This parameter controls the size (in kBytes) of packed revprop files. +### Revprops of consecutive revisions will be concatenated into a single +### file up to but not exceeding the threshold given here. However, each +### pack file may be much smaller and revprops of a single revision may be +### much larger than the limit set here. The threshold will be applied +### before optional compression takes place. +### Large values will reduce disk space usage at the expense of increased +### latency and CPU usage reading and changing individual revprops. +### Values smaller than 4 kByte will not improve latency any further and +### quickly render revprop packing ineffective. +### revprop-pack-size is 16 kBytes by default for non-compressed revprop +### pack files and 64 kBytes when compression has been enabled. +# revprop-pack-size = 16 +### +### To save disk space, packed revprop files may be compressed. Standard +### revprops tend to allow for very effective compression. Reading and +### even more so writing, become significantly more CPU intensive. +### Compressing packed revprops is disabled by default. +# compress-packed-revprops = false + +[io] +### Parameters in this section control the data access granularity in +### format 7 repositories and later. The defaults should translate into +### decent performance over a wide range of setups. +### +### When a specific piece of information needs to be read from disk, a +### data block is being read at once and its contents are being cached. +### If the repository is being stored on a RAID, the block size should be +### either 50% or 100% of RAID block size / granularity. Also, your file +### system blocks/clusters should be properly aligned and sized. In that +### setup, each access will hit only one disk (minimizes I/O load) but +### uses all the data provided by the disk in a single access. +### For SSD-based storage systems, slightly lower values around 16 kB +### may improve latency while still maximizing throughput. If block-read +### has not been enabled, this will be capped to 4 kBytes. +### Can be changed at any time but must be a power of 2. +### block-size is given in kBytes and with a default of 64 kBytes. +# block-size = 64 +### +### The log-to-phys index maps data item numbers to offsets within the +### rev or pack file. This index is organized in pages of a fixed maximum +### capacity. To access an item, the page table and the respective page +### must be read. +### This parameter only affects revisions with thousands of changed paths. +### If you have several extremely large revisions (~1 mio changes), think +### about increasing this setting. Reducing the value will rarely result +### in a net speedup. +### This is an expert setting. Must be a power of 2. +### l2p-page-size is 8192 entries by default. +# l2p-page-size = 8192 +### +### The phys-to-log index maps positions within the rev or pack file to +### to data items, i.e. describes what piece of information is being +### stored at any particular offset. The index describes the rev file +### in chunks (pages) and keeps a global list of all those pages. Large +### pages mean a shorter page table but a larger per-page description of +### data items in it. The latency sweetspot depends on the change size +### distribution but covers a relatively wide range. +### If the repository contains very large files, i.e. individual changes +### of tens of MB each, increasing the page size will shorten the index +### file at the expense of a slightly increased latency in sections with +### smaller changes. +### For source code repositories, this should be about 16x the block-size. +### Must be a power of 2. +### p2l-page-size is given in kBytes and with a default of 1024 kBytes. +# p2l-page-size = 1024 + +[debug] +### +### Whether to verify each new revision immediately before finalizing +### the commit. This is disabled by default except in maintainer-mode +### builds. +# verify-before-commit = false diff --git a/test/repos/simple-ext.svn/db/min-unpacked-rev b/test/repos/simple-ext.svn/db/min-unpacked-rev new file mode 100644 index 0000000000..573541ac97 --- /dev/null +++ b/test/repos/simple-ext.svn/db/min-unpacked-rev @@ -0,0 +1 @@ +0 diff --git a/test/repos/simple-ext.svn/db/rep-cache.db b/test/repos/simple-ext.svn/db/rep-cache.db new file mode 100644 index 0000000000000000000000000000000000000000..3193b2eaad046fba3704b0d28a50a68a94801da2 GIT binary patch literal 8192 zcmeIuK}!Nb6bJAbjiM;*RyW_VAR=4M^_JC;OPgEUhQTy(n&njm5*>#d|ehiI!uT zo%TC>#d3w1Jtzo300Izz00bZa0SG_<0uX?}pA`5U@~wkvm49vLDta+zt1RRiHZQ=L;CU@@4?#yhXLbg~~mc>fT`3nX?Ls&t(q_@;kd&|6q zPmnj!N-"j%`;)7F<&!^&p5-&NVNc`P`LVZ4M1vhI123LfW679Jk&yS}h4J?+gU zS3MZfY8^$e`Aiw@d{Ry%1dI`+SBZs%>uw7^VT{$-o>k4Xyk=x|K`9*$$d+#CX^Zv$ zf4Az(2{}^ad;78;-W-sx-=Fn>9+LNu&)3o0-}x~i0yI}C8O(!53mF)Lc9yx|g^G|c ej~cmPzoZI_YZ6{Gdr_88^Nk1hnf(AO7WJ=&dj!fDn0uL2;xcFgCGi};7P%MLA>_hL1!bWwMgk<=JI{-``-8Sx~2ler?X@>qvZhS z>jPT>)6KB%uk~`LVLed!QU%IMrh0nGt~zC~p7r~M2xW}B&Vh{`_(=}ATKsQ!Fzyy7 zc4uq7<>SMvwZ?-a) z)0B>oW87dZf4*6*J;dJdmgjkw&ZC`k2j2VA6K~^ji#5jLtJ2Wv5xP@BFE&4IU%WYq zems5i{p{Q%din14uL~NZ-LD^~H)6E)d2he;%)5-9fBCR^<=fA>YX}J*yI_JaqN6CX rO2(X~(lQ>UT$426lExq+3DoM8C~b*QiI&DOO{6l~rdCtJNj(1>+FjA$ literal 0 HcmV?d00001 diff --git a/test/repos/simple-ext.svn/db/revs/0/2 b/test/repos/simple-ext.svn/db/revs/0/2 new file mode 100644 index 0000000000000000000000000000000000000000..99a14cf4b7f76ee26eba753930dd53704627e1f5 GIT binary patch literal 816 zcmZvZJ!lj`6vt;bpA3nm2sW<~jBwfc+L_sFOu`))jUhw{0gGh577vfyac(dHi+~?R zh?R}CT|}_e+9r*qV5N;MStC{G3)m;lNk zCm~lv(?C<68Ar-%YL%o`uYzC#rkyZdVR9kD9Q-!%}%S;O#unUwLYMU_)@exaSbTJeGis^rnsBF zpQ$A6r{xuTL@SSJeWQcholp+qxI~k*M^FFKr;u_G6ohjsvpQAEc$GQHnKxDmExDoA zqy#EZIZ5|4d3k=%Gb>dAc}$Wl&XYP%bKGo< zTmjWejYqZH7n7Q%0^8yGT2lbUjl12Rz=7w$2rz4bdULXUmfC*4ic;URV zT5%(!r#bby^on`qSY{66?ef&aVF)RMbPpw2dC8>a+6zOarB*P`g_V_4QdROm3)2?cKX5He8X8kD zRTa09dUI>+4($Ie?{8&&x)?7E=6hH1%;vc$isp`@N5_6ekGpR>52FX2McUt&KhAE3 z8yY={3bg&={dRN#qtCB?emH(VI)R>Tecg&qWAyFm%XhPPH2SpjWe4}>(DxsApLgH< zK6?@&t~A)(Gf_HKN~e|J&g9-2E__xI=82{tXa=RNLR*{|XCxC^W?VX{y(YOLf>dnb E4; /dev/null || exit 1 + +# Check that the author of this commit has the rights to perform +# the commit on the files and directories being modified. +commit-access-control.pl "$REPOS" "$TXN" commit-access-control.cfg || exit 1 + +# All checks passed, so allow the commit. +exit 0 diff --git a/test/repos/simple-ext.svn/hooks/pre-lock.tmpl b/test/repos/simple-ext.svn/hooks/pre-lock.tmpl new file mode 100755 index 0000000000..148582a689 --- /dev/null +++ b/test/repos/simple-ext.svn/hooks/pre-lock.tmpl @@ -0,0 +1,95 @@ +#!/bin/sh + +# PRE-LOCK HOOK +# +# The pre-lock hook is invoked before an exclusive lock is +# created. Subversion runs this hook by invoking a program +# (script, executable, binary, etc.) named 'pre-lock' (for which +# this file is a template), with the following ordered arguments: +# +# [1] REPOS-PATH (the path to this repository) +# [2] PATH (the path in the repository about to be locked) +# [3] USER (the user creating the lock) +# [4] COMMENT (the comment of the lock) +# [5] STEAL-LOCK (1 if the user is trying to steal the lock, else 0) +# +# If the hook program outputs anything on stdout, the output string will +# be used as the lock token for this lock operation. If you choose to use +# this feature, you must guarantee the tokens generated are unique across +# the repository each time. +# +# If the hook program exits with success, the lock is created; but +# if it exits with failure (non-zero), the lock action is aborted +# and STDERR is returned to the client. +# +# The default working directory for the invocation is undefined, so +# the program should set one explicitly if it cares. +# +# On a Unix system, the normal procedure is to have 'pre-lock' +# invoke other programs to do the real work, though it may do the +# work itself too. +# +# Note that 'pre-lock' must be executable by the user(s) who will +# invoke it (typically the user httpd runs as), and that user must +# have filesystem-level permission to access the repository. +# +# On a Windows system, you should name the hook program +# 'pre-lock.bat' or 'pre-lock.exe', +# but the basic idea is the same. +# +# The hook program runs in an empty environment, unless the server is +# explicitly configured otherwise. For example, a common problem is for +# the PATH environment variable to not be set to its usual value, so +# that subprograms fail to launch unless invoked via absolute path. +# If you're having unexpected problems with a hook program, the +# culprit may be unusual (or missing) environment variables. +# +# CAUTION: +# For security reasons, you MUST always properly quote arguments when +# you use them, as those arguments could contain whitespace or other +# problematic characters. Additionally, you should delimit the list +# of options with "--" before passing the arguments, so malicious +# clients cannot bootleg unexpected options to the commands your +# script aims to execute. +# For similar reasons, you should also add a trailing @ to URLs which +# are passed to SVN commands accepting URLs with peg revisions. +# +# Here is an example hook script, for a Unix /bin/sh interpreter. +# For more examples and pre-written hooks, see those in +# the Subversion repository at +# http://svn.apache.org/repos/asf/subversion/trunk/tools/hook-scripts/ and +# http://svn.apache.org/repos/asf/subversion/trunk/contrib/hook-scripts/ + + +REPOS="$1" +PATH="$2" +USER="$3" +COMMENT="$4" +STEAL="$5" + +# If a lock exists and is owned by a different person, don't allow it +# to be stolen (e.g., with 'svn lock --force ...'). + +# (Maybe this script could send email to the lock owner?) +SVNLOOK=/opt/homebrew/Cellar/subversion/1.14.2_1/bin/svnlook +GREP=/bin/grep +SED=/bin/sed + +LOCK_OWNER=`$SVNLOOK lock "$REPOS" "$PATH" | \ + $GREP '^Owner: ' | $SED 's/Owner: //'` + +# If we get no result from svnlook, there's no lock, allow the lock to +# happen: +if [ "$LOCK_OWNER" = "" ]; then + exit 0 +fi + +# If the person locking matches the lock's owner, allow the lock to +# happen: +if [ "$LOCK_OWNER" = "$USER" ]; then + exit 0 +fi + +# Otherwise, we've got an owner mismatch, so return failure: +echo "Error: $PATH already locked by ${LOCK_OWNER}." 1>&2 +exit 1 diff --git a/test/repos/simple-ext.svn/hooks/pre-revprop-change.tmpl b/test/repos/simple-ext.svn/hooks/pre-revprop-change.tmpl new file mode 100755 index 0000000000..8b065d7c79 --- /dev/null +++ b/test/repos/simple-ext.svn/hooks/pre-revprop-change.tmpl @@ -0,0 +1,79 @@ +#!/bin/sh + +# PRE-REVPROP-CHANGE HOOK +# +# The pre-revprop-change hook is invoked before a revision property +# is added, modified or deleted. Subversion runs this hook by invoking +# a program (script, executable, binary, etc.) named 'pre-revprop-change' +# (for which this file is a template), with the following ordered +# arguments: +# +# [1] REPOS-PATH (the path to this repository) +# [2] REV (the revision being tweaked) +# [3] USER (the username of the person tweaking the property) +# [4] PROPNAME (the property being set on the revision) +# [5] ACTION (the property is being 'A'dded, 'M'odified, or 'D'eleted) +# +# [STDIN] PROPVAL ** the new property value is passed via STDIN. +# +# If the hook program exits with success, the propchange happens; but +# if it exits with failure (non-zero), the propchange doesn't happen. +# The hook program can use the 'svnlook' utility to examine the +# existing value of the revision property. +# +# WARNING: unlike other hooks, this hook MUST exist for revision +# properties to be changed. If the hook does not exist, Subversion +# will behave as if the hook were present, but failed. The reason +# for this is that revision properties are UNVERSIONED, meaning that +# a successful propchange is destructive; the old value is gone +# forever. We recommend the hook back up the old value somewhere. +# +# The default working directory for the invocation is undefined, so +# the program should set one explicitly if it cares. +# +# On a Unix system, the normal procedure is to have 'pre-revprop-change' +# invoke other programs to do the real work, though it may do the +# work itself too. +# +# Note that 'pre-revprop-change' must be executable by the user(s) who will +# invoke it (typically the user httpd runs as), and that user must +# have filesystem-level permission to access the repository. +# +# On a Windows system, you should name the hook program +# 'pre-revprop-change.bat' or 'pre-revprop-change.exe', +# but the basic idea is the same. +# +# The hook program runs in an empty environment, unless the server is +# explicitly configured otherwise. For example, a common problem is for +# the PATH environment variable to not be set to its usual value, so +# that subprograms fail to launch unless invoked via absolute path. +# If you're having unexpected problems with a hook program, the +# culprit may be unusual (or missing) environment variables. +# +# CAUTION: +# For security reasons, you MUST always properly quote arguments when +# you use them, as those arguments could contain whitespace or other +# problematic characters. Additionally, you should delimit the list +# of options with "--" before passing the arguments, so malicious +# clients cannot bootleg unexpected options to the commands your +# script aims to execute. +# For similar reasons, you should also add a trailing @ to URLs which +# are passed to SVN commands accepting URLs with peg revisions. +# +# Here is an example hook script, for a Unix /bin/sh interpreter. +# For more examples and pre-written hooks, see those in +# the Subversion repository at +# http://svn.apache.org/repos/asf/subversion/trunk/tools/hook-scripts/ and +# http://svn.apache.org/repos/asf/subversion/trunk/contrib/hook-scripts/ + + +REPOS="$1" +REV="$2" +USER="$3" +PROPNAME="$4" +ACTION="$5" + +if [ "$ACTION" = "M" -a "$PROPNAME" = "svn:log" ]; then exit 0; fi + +echo "Changing revision properties other than svn:log is prohibited" >&2 +exit 1 diff --git a/test/repos/simple-ext.svn/hooks/pre-unlock.tmpl b/test/repos/simple-ext.svn/hooks/pre-unlock.tmpl new file mode 100755 index 0000000000..9ba99d071b --- /dev/null +++ b/test/repos/simple-ext.svn/hooks/pre-unlock.tmpl @@ -0,0 +1,87 @@ +#!/bin/sh + +# PRE-UNLOCK HOOK +# +# The pre-unlock hook is invoked before an exclusive lock is +# destroyed. Subversion runs this hook by invoking a program +# (script, executable, binary, etc.) named 'pre-unlock' (for which +# this file is a template), with the following ordered arguments: +# +# [1] REPOS-PATH (the path to this repository) +# [2] PATH (the path in the repository about to be unlocked) +# [3] USER (the user destroying the lock) +# [4] TOKEN (the lock token to be destroyed) +# [5] BREAK-UNLOCK (1 if the user is breaking the lock, else 0) +# +# If the hook program exits with success, the lock is destroyed; but +# if it exits with failure (non-zero), the unlock action is aborted +# and STDERR is returned to the client. +# +# The default working directory for the invocation is undefined, so +# the program should set one explicitly if it cares. +# +# On a Unix system, the normal procedure is to have 'pre-unlock' +# invoke other programs to do the real work, though it may do the +# work itself too. +# +# Note that 'pre-unlock' must be executable by the user(s) who will +# invoke it (typically the user httpd runs as), and that user must +# have filesystem-level permission to access the repository. +# +# On a Windows system, you should name the hook program +# 'pre-unlock.bat' or 'pre-unlock.exe', +# but the basic idea is the same. +# +# The hook program runs in an empty environment, unless the server is +# explicitly configured otherwise. For example, a common problem is for +# the PATH environment variable to not be set to its usual value, so +# that subprograms fail to launch unless invoked via absolute path. +# If you're having unexpected problems with a hook program, the +# culprit may be unusual (or missing) environment variables. +# +# CAUTION: +# For security reasons, you MUST always properly quote arguments when +# you use them, as those arguments could contain whitespace or other +# problematic characters. Additionally, you should delimit the list +# of options with "--" before passing the arguments, so malicious +# clients cannot bootleg unexpected options to the commands your +# script aims to execute. +# For similar reasons, you should also add a trailing @ to URLs which +# are passed to SVN commands accepting URLs with peg revisions. +# +# Here is an example hook script, for a Unix /bin/sh interpreter. +# For more examples and pre-written hooks, see those in +# the Subversion repository at +# http://svn.apache.org/repos/asf/subversion/trunk/tools/hook-scripts/ and +# http://svn.apache.org/repos/asf/subversion/trunk/contrib/hook-scripts/ + + +REPOS="$1" +PATH="$2" +USER="$3" +TOKEN="$4" +BREAK="$5" + +# If a lock is owned by a different person, don't allow it be broken. +# (Maybe this script could send email to the lock owner?) + +SVNLOOK=/opt/homebrew/Cellar/subversion/1.14.2_1/bin/svnlook +GREP=/bin/grep +SED=/bin/sed + +LOCK_OWNER=`$SVNLOOK lock "$REPOS" "$PATH" | \ + $GREP '^Owner: ' | $SED 's/Owner: //'` + +# If we get no result from svnlook, there's no lock, return success: +if [ "$LOCK_OWNER" = "" ]; then + exit 0 +fi + +# If the person unlocking matches the lock's owner, return success: +if [ "$LOCK_OWNER" = "$USER" ]; then + exit 0 +fi + +# Otherwise, we've got an owner mismatch, so return failure: +echo "Error: $PATH locked by ${LOCK_OWNER}." 1>&2 +exit 1 diff --git a/test/repos/simple-ext.svn/hooks/start-commit.tmpl b/test/repos/simple-ext.svn/hooks/start-commit.tmpl new file mode 100755 index 0000000000..1395e8315a --- /dev/null +++ b/test/repos/simple-ext.svn/hooks/start-commit.tmpl @@ -0,0 +1,81 @@ +#!/bin/sh + +# START-COMMIT HOOK +# +# The start-commit hook is invoked immediately after a Subversion txn is +# created and populated with initial revprops in the process of doing a +# commit. Subversion runs this hook by invoking a program (script, +# executable, binary, etc.) named 'start-commit' (for which this file +# is a template) with the following ordered arguments: +# +# [1] REPOS-PATH (the path to this repository) +# [2] USER (the authenticated user attempting to commit) +# [3] CAPABILITIES (a colon-separated list of capabilities reported +# by the client; see note below) +# [4] TXN-NAME (the name of the commit txn just created) +# +# Note: The CAPABILITIES parameter is new in Subversion 1.5, and 1.5 +# clients will typically report at least the "mergeinfo" capability. +# If there are other capabilities, then the list is colon-separated, +# e.g.: "mergeinfo:some-other-capability" (the order is undefined). +# +# The list is self-reported by the client. Therefore, you should not +# make security assumptions based on the capabilities list, nor should +# you assume that clients reliably report every capability they have. +# +# Note: The TXN-NAME parameter is new in Subversion 1.8. Prior to version +# 1.8, the start-commit hook was invoked before the commit txn was even +# created, so the ability to inspect the commit txn and its metadata from +# within the start-commit hook was not possible. +# +# If the hook program exits with success, the commit continues; but +# if it exits with failure (non-zero), the commit is stopped before +# a Subversion txn is created, and STDERR is returned to the client. +# +# The default working directory for the invocation is undefined, so +# the program should set one explicitly if it cares. +# +# On a Unix system, the normal procedure is to have 'start-commit' +# invoke other programs to do the real work, though it may do the +# work itself too. +# +# Note that 'start-commit' must be executable by the user(s) who will +# invoke it (typically the user httpd runs as), and that user must +# have filesystem-level permission to access the repository. +# +# On a Windows system, you should name the hook program +# 'start-commit.bat' or 'start-commit.exe', +# but the basic idea is the same. +# +# The hook program runs in an empty environment, unless the server is +# explicitly configured otherwise. For example, a common problem is for +# the PATH environment variable to not be set to its usual value, so +# that subprograms fail to launch unless invoked via absolute path. +# If you're having unexpected problems with a hook program, the +# culprit may be unusual (or missing) environment variables. +# +# CAUTION: +# For security reasons, you MUST always properly quote arguments when +# you use them, as those arguments could contain whitespace or other +# problematic characters. Additionally, you should delimit the list +# of options with "--" before passing the arguments, so malicious +# clients cannot bootleg unexpected options to the commands your +# script aims to execute. +# For similar reasons, you should also add a trailing @ to URLs which +# are passed to SVN commands accepting URLs with peg revisions. +# +# Here is an example hook script, for a Unix /bin/sh interpreter. +# For more examples and pre-written hooks, see those in +# the Subversion repository at +# http://svn.apache.org/repos/asf/subversion/trunk/tools/hook-scripts/ and +# http://svn.apache.org/repos/asf/subversion/trunk/contrib/hook-scripts/ + + +REPOS="$1" +USER="$2" + +commit-allower.pl --repository "$REPOS" --user "$USER" || exit 1 +special-auth-check.py --user "$USER" --auth-level 3 || exit 1 + +# All checks passed, so allow the commit. +exit 0 diff --git a/test/repos/simple-ext.svn/locks/db-logs.lock b/test/repos/simple-ext.svn/locks/db-logs.lock new file mode 100644 index 0000000000..20dd6369be --- /dev/null +++ b/test/repos/simple-ext.svn/locks/db-logs.lock @@ -0,0 +1,3 @@ +This file is not used by Subversion 1.3.x or later. +However, its existence is required for compatibility with +Subversion 1.2.x or earlier. diff --git a/test/repos/simple-ext.svn/locks/db.lock b/test/repos/simple-ext.svn/locks/db.lock new file mode 100644 index 0000000000..20dd6369be --- /dev/null +++ b/test/repos/simple-ext.svn/locks/db.lock @@ -0,0 +1,3 @@ +This file is not used by Subversion 1.3.x or later. +However, its existence is required for compatibility with +Subversion 1.2.x or earlier. diff --git a/test/test_sys_checkout.py b/test/test_sys_checkout.py old mode 100644 new mode 100755 index ab4f77e88f..664160dc99 --- a/test/test_sys_checkout.py +++ b/test/test_sys_checkout.py @@ -97,6 +97,7 @@ SIMPLE_REPO = 'simple-ext.git' # Child repo SIMPLE_FORK_REPO = 'simple-ext-fork.git' # Child repo MIXED_REPO = 'mixed-cont-ext.git' # Both parent and child +SVN_TEST_REPO = 'simple-ext.svn' # Subversion repository # Standard (arbitrary) external names for test configs TAG_SECTION = 'simp_tag' @@ -120,8 +121,6 @@ # Branch that exists in both the simple and simple-fork repos. REMOTE_BRANCH_FEATURE2 = 'feature2' -SVN_TEST_REPO = 'https://github.com/escomp/cesm' - # Disable too-many-public-methods error # pylint: disable=R0904 @@ -354,7 +353,7 @@ def create_section_reference_to_subexternal(self, name): self._config.set(name, ExternalsDescription.EXTERNALS, CFG_SUB_NAME) - def create_svn_external(self, name, tag='', branch=''): + def create_svn_external(self, name, url, tag='', branch=''): """Create a config section for an svn repository. """ @@ -365,7 +364,7 @@ def create_svn_external(self, name, tag='', branch=''): self._config.set(name, ExternalsDescription.PROTOCOL, ExternalsDescription.PROTOCOL_SVN) - self._config.set(name, ExternalsDescription.REPO_URL, SVN_TEST_REPO) + self._config.set(name, ExternalsDescription.REPO_URL, url) self._config.set(name, ExternalsDescription.REQUIRED, str(True)) @@ -1387,36 +1386,10 @@ def test_container_sparse(self): 'simple_subdir', 'subdir_file.txt')) - class TestSysCheckoutSVN(BaseTestSysCheckout): """Run systems level tests of checkout_externals accessing svn repositories - SVN tests - these tests use the svn repository interface. Since - they require an active network connection, they are significantly - slower than the git tests. But svn testing is critical. So try to - design the tests to only test svn repository functionality - (checkout, switch) and leave generic testing of functionality like - 'optional' to the fast git tests. - - Example timing as of 2017-11: - - * All other git and unit tests combined take between 4-5 seconds - - * Just checking if svn is available for a single test takes 2 seconds. - - * The single svn test typically takes between 10 and 25 seconds - (depending on the network)! - - NOTE(bja, 2017-11) To enable CI testing we can't use a real remote - repository that restricts access and it seems inappropriate to hit - a random open source repo. For now we are just hitting one of our - own github repos using the github svn server interface. This - should be "good enough" for basic checkout and swich - functionality. But if additional svn functionality is required, a - better solution will be necessary. I think eventually we want to - create a small local svn repository on the fly (doesn't require an - svn server or network connection!) and use it for testing. - + SVN tests - these tests use the svn repository interface. """ @staticmethod @@ -1427,6 +1400,9 @@ def _svn_branch_name(): def _svn_tag_name(): return './{0}/svn_tag'.format(EXTERNALS_PATH) + def _svn_test_repo_url(self): + return 'file://' + os.path.join(self._bare_root, SVN_TEST_REPO) + def _check_tag_branch_svn_tag_clean(self, tree): self._check_sync_clean(tree[self._external_path(TAG_SECTION)], ExternalStatus.STATUS_OK, @@ -1438,13 +1414,12 @@ def _check_tag_branch_svn_tag_clean(self, tree): ExternalStatus.STATUS_OK, ExternalStatus.STATUS_OK) - @staticmethod - def _have_svn_access(): + def _have_svn_access(self): """Check if we have svn access so we can enable tests that use svn. """ have_svn = False - cmd = ['svn', 'ls', SVN_TEST_REPO, ] + cmd = ['svn', 'ls', self._svn_test_repo_url(), ] try: execute_subprocess(cmd) have_svn = True @@ -1472,8 +1447,8 @@ def test_container_simple_svn(self): self._generator.create_section(SIMPLE_REPO, TAG_SECTION, tag='tag1') # Svn repos. - self._generator.create_svn_external('svn_branch', branch='trunk') - self._generator.create_svn_external('svn_tag', tag='tags/cesm2.0.beta07') + self._generator.create_svn_external('svn_branch', self._svn_test_repo_url(), branch='trunk') + self._generator.create_svn_external('svn_tag', self._svn_test_repo_url(), tag='tags/cesm2.0.beta07') self._generator.write_config(cloned_repo_dir) @@ -1557,7 +1532,7 @@ def setUp(self): execute_subprocess(cmd) cmd = ['git', 'checkout', self._bare_branch_name] execute_subprocess(cmd) - cmd = ['git', 'submodule', 'add', fork_repo_dir] + cmd = ['git', '-c', 'protocol.file.allow=always','submodule', 'add', fork_repo_dir] execute_subprocess(cmd) cmd = ['git', 'commit', '-am', "'Added simple-ext-fork as a submodule'"] execute_subprocess(cmd) @@ -1571,7 +1546,7 @@ def setUp(self): execute_subprocess(cmd) cmd = ['git', 'checkout', self._config_branch_name] execute_subprocess(cmd) - cmd = ['git', 'submodule', 'add', '--name', SIMPLE_REPO, + cmd = ['git', '-c', 'protocol.file.allow=always', 'submodule', 'add', '--name', SIMPLE_REPO, simple_repo_dir, self._simple_ext_name] execute_subprocess(cmd) # Checkout feature2 From c39526714d740f2bebbcedad0b5d41a293491cef Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 21 Feb 2024 11:14:18 -0700 Subject: [PATCH 03/28] Update expected fails to remove tests that I now expect to pass --- cime_config/testdefs/ExpectedTestFails.xml | 55 ---------------------- 1 file changed, 55 deletions(-) diff --git a/cime_config/testdefs/ExpectedTestFails.xml b/cime_config/testdefs/ExpectedTestFails.xml index 55d1363e6b..a82c8e4323 100644 --- a/cime_config/testdefs/ExpectedTestFails.xml +++ b/cime_config/testdefs/ExpectedTestFails.xml @@ -29,12 +29,6 @@ - - - FAIL - #2268 - - @@ -58,41 +52,6 @@ - - - FAIL - ESMCI/ccs_config_cesm#131 - - - - - - FAIL - ESMCI/ccs_config_cesm#130 - - - - - - FAIL - ESMCI/ccs_config_cesm#130 - - - - - - FAIL - ESMCI/ccs_config_cesm#130 - - - - - - FAIL - ESMCI/ccs_config_cesm#130 - - - FAIL @@ -192,20 +151,6 @@ - - - FAIL - ESMCI/ccs_config_cesm#130 - - - - - - FAIL - ESMCI/ccs_config_cesm#130 - - - PEND From 19d39d98949c21b3b97753b211cbbc05fff1294c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 21 Feb 2024 11:18:52 -0700 Subject: [PATCH 04/28] Add back intel/debug with mpi-serial tests on Derecho as they should now work --- cime_config/testdefs/testlist_clm.xml | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index d8ef6357cd..0c8bd9a4ff 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1984,10 +1984,8 @@ - @@ -2108,9 +2106,7 @@ - @@ -2123,9 +2119,7 @@ - @@ -2137,9 +2131,7 @@ - @@ -2174,10 +2166,8 @@ - @@ -2193,10 +2183,8 @@ - @@ -2604,10 +2592,8 @@ - @@ -2627,9 +2613,7 @@ - @@ -2965,9 +2949,7 @@ - @@ -2994,9 +2976,7 @@ - From 76c23ec0ca998784b370f7a2ddfdb9e8ec667fb5 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 21 Feb 2024 13:50:12 -0700 Subject: [PATCH 05/28] Changes to remove difference in CGD conda environment vs Derecho, also needed to update numpy version for it to work --- py_env_create | 19 +++---------------- python/README.md | 1 - python/conda_env_ctsm_py.txt | 6 ++---- python/conda_env_ctsm_py_cgd.txt | 24 ------------------------ 4 files changed, 5 insertions(+), 45 deletions(-) delete mode 100644 python/conda_env_ctsm_py_cgd.txt diff --git a/py_env_create b/py_env_create index c323a374df..4b3612cfda 100755 --- a/py_env_create +++ b/py_env_create @@ -15,8 +15,8 @@ conda --help >& condahelp.txt error=$? if [ $error != 0 ]; then echo "conda is NOT in your path for the bash shell add it with modules or whatever is required on your system to get it in your path" - echo "on cheyenne/capser/etc use -- module load conda" - echo "on izumi/CGD systems use -- module load lang/python" + echo "on Derecho/capser/etc use -- module load conda" + echo "on izumi/CGD systems use -- module unload lang/python; module load lang/anaconda/23.11.0/base" echo "For notes on installing on a user system see: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html" echo "Error code was $error" cat condahelp.txt @@ -29,11 +29,7 @@ ctsm_python=ctsm_pylib condadir="$dir/python" domain=`domainname` -if [[ $domain =~ cgd.* ]]; then - condafile="conda_env_ctsm_py_cgd.txt" -else - condafile="conda_env_ctsm_py.txt" -fi +condafile="conda_env_ctsm_py.txt" #---------------------------------------------------------------------- # Usage subroutine usage() { @@ -119,15 +115,6 @@ if [ $? != 0 ]; then echo "Trouble installing the $ctsm_python python environment" echo "There must be a problem in the $condadir/$condafile conda specification environment file" echo "Change the file and try again" - if [[ $domain =~ cgd.* ]]; then - pythonpath=`which python` - echo - echo "On CGD systems you may need to do the following..." - echo "Create a bin subdirectory and then link the python version into it..." - echo " mkdir $HOME/.conda/envs/$ctsm_python/bin/" - echo " ln -s $pythonpath $HOME/.conda/envs/$ctsm_python/bin/python3.7" - echo - fi exit -2 fi echo "Successfully installed the $ctsm_python python environment" diff --git a/python/README.md b/python/README.md index c40f55c6c7..d0024744e4 100644 --- a/python/README.md +++ b/python/README.md @@ -21,7 +21,6 @@ all the conda commands and do this for you. Conda requirements files: conda_env_ctsm_py.txt --------- Standard conda environment to use for most machines -conda_env_ctsm_py_cgd.txt ----- Standard conda environment for CGD machines conda_env_ctsm_py_latest.txt -- Test environment with latest versions that work ## Unit and system tests diff --git a/python/conda_env_ctsm_py.txt b/python/conda_env_ctsm_py.txt index e621081591..abccf74773 100644 --- a/python/conda_env_ctsm_py.txt +++ b/python/conda_env_ctsm_py.txt @@ -1,7 +1,5 @@ # -# NOTE: Changes here should be coordinated with the cgd python environment file -# -# NOTE: Derecho already has conda installed for you, so you just need to do the following... +# NOTE: On Derecho you may need to "module load conda" # # use the top level bash script: # ../py_env_create # Do this each time you update your CTSM Version @@ -18,7 +16,7 @@ scipy netcdf4 requests packaging -numpy=1.18.5 +numpy=1.19.5 xarray=0.17.0 xesmf numba=0.55.2 # Avoid 0.56 until numpy>=1.20. This is the minimum for xesmf diff --git a/python/conda_env_ctsm_py_cgd.txt b/python/conda_env_ctsm_py_cgd.txt deleted file mode 100644 index 3afcf4bba2..0000000000 --- a/python/conda_env_ctsm_py_cgd.txt +++ /dev/null @@ -1,24 +0,0 @@ -# NOTE: This version is just for cgd.ucar.edu systems, where conda limits python to version 3.7.0 -# See issue https://github.com/ESCOMP/CTSM/issues/1792 -# -# This should be coordinated with the main python environment file! -# -# use the top level bash script: -# ../py_env_create # Do this each time you update your CTSM Version -# conda activate ctsm_pylib # Do this anytime you want to run a CTSM python script -# Or the individual conda commands: -# conda create -n ctsm_pylib # Do this one time for each machine -# conda install -n ctsm_pylib --file conda_env_ctsm_py.txt # Do this each time you update your CTSM Version -# conda activate ctsm_pylib # Do this anytime you want to run a CTSM python script -# -python=3.7.0 # The python version MUST match the python version available on CGD systems through modules exactly -pandas -tqdm -scipy -netcdf4 -requests -packaging -numpy=1.18.5 -xarray=0.17.0 -pylint=2.8.3 -black=22.3.0 # NOTE: The version here needs to be coordinated with the black github action under ../.github/workflows From df6d9afa54f29126ef7f2389bb2ce949b5f01dd4 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 21 Feb 2024 22:27:25 -0700 Subject: [PATCH 06/28] Fix queue for Derecho which should be unspecified, the derecho queues are main and develop, with the new externals the testlist wasn't working, it failed trying to setup cases --- python/ctsm/machine_defaults.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ctsm/machine_defaults.py b/python/ctsm/machine_defaults.py index 0f3900c152..a17f063a4b 100644 --- a/python/ctsm/machine_defaults.py +++ b/python/ctsm/machine_defaults.py @@ -74,7 +74,7 @@ baseline_dir=os.path.join(os.path.sep, "glade", "campaign", "cgd", "tss", "ctsm_baselines"), account_required=True, create_test_retry=0, - create_test_queue="regular", + create_test_queue=CREATE_TEST_QUEUE_UNSPECIFIED, job_launcher_defaults={ JOB_LAUNCHER_QSUB: QsubDefaults( queue="main", From 5f5e12e9600c4c29d89a7aff001578d0c7128124 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 22 Feb 2024 19:34:58 -0700 Subject: [PATCH 07/28] Update to a cime branch that fixes the ERI test issues --- Externals.cfg | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 21499fc452..4650869d55 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -43,8 +43,10 @@ required = True [cime] local_path = cime protocol = git -repo_url = https://github.com/ESMCI/cime -tag = cime6.0.217_httpsbranch01 +#repo_url = https://github.com/ESMCI/cime +repo_url = https://github.com/ekluzek/cime +#tag = cime6.0.217_httpsbranch01 +hash = a20fcb6a52c830096d4fa388873a3505f6897368 required = True [cmeps] From 001829b352130012b374e1ba77f94d49d42ce27c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 22 Feb 2024 23:39:49 -0700 Subject: [PATCH 08/28] Update to the tag on the CESM branch of cime that fixes the ERI problem --- Externals.cfg | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 4650869d55..3442bb7bfa 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -43,10 +43,8 @@ required = True [cime] local_path = cime protocol = git -#repo_url = https://github.com/ESMCI/cime -repo_url = https://github.com/ekluzek/cime -#tag = cime6.0.217_httpsbranch01 -hash = a20fcb6a52c830096d4fa388873a3505f6897368 +repo_url = https://github.com/ESMCI/cime +tag = cime6.0.217_httpsbranch02 required = True [cmeps] From 43c58e2d3369e431d1598b49ad3d405ff019871a Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 26 Feb 2024 11:34:03 -0700 Subject: [PATCH 09/28] conf.py: Change add_stylesheet() to add_css_file(). The former is now breaking my ability to build the docs in VSCode. It seems to have no impact on the ability to build the docs using the Docker method. --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index dcd0b2eae6..6c00f5a686 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -189,4 +189,4 @@ numfig_secnum_depth = 2 def setup(app): - app.add_stylesheet('css/custom.css') + app.add_css_file('css/custom.css') From 82db210ca17c00576eb3ab780ac4b64422da78df Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 26 Feb 2024 11:34:20 -0700 Subject: [PATCH 10/28] Fix a table in Methane tech note. --- doc/source/tech_note/Methane/CLM50_Tech_Note_Methane.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/tech_note/Methane/CLM50_Tech_Note_Methane.rst b/doc/source/tech_note/Methane/CLM50_Tech_Note_Methane.rst index 90be6e6ad0..ac45114a28 100644 --- a/doc/source/tech_note/Methane/CLM50_Tech_Note_Methane.rst +++ b/doc/source/tech_note/Methane/CLM50_Tech_Note_Methane.rst @@ -225,7 +225,7 @@ For gaseous diffusion, we adopted the temperature dependence of molecular free-a +==========================================================+==========================================================+========================================================+ | Aqueous | 0.9798 + 0.02986\ *T* + 0.0004381\ *T*\ :sup:`2` | 1.172+ 0.03443\ *T* + 0.0005048\ *T*\ :sup:`2` | +----------------------------------------------------------+----------------------------------------------------------+--------------------------------------------------------+ - | Gaseous | 0.1875 + 0.0013\ *T* | 0.1759 + 0.00117\ *T* | + | Gaseous | 0.1875 + 0.0013\ *T* | 0.1759 + 0.00117\ *T* | +----------------------------------------------------------+----------------------------------------------------------+--------------------------------------------------------+ Gaseous diffusivity in soils also depends on the molecular diffusivity, soil structure, porosity, and organic matter content. :ref:`Moldrup et al. (2003)`, using observations across a range of unsaturated mineral soils, showed that the relationship between effective diffusivity (:math:`D_{e}` (m\ :sup:`2` s\ :sup:`-1`)) and soil properties can be represented as: From ca6237ae02718e29b89a9914ed9f978f42d31ee4 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 26 Feb 2024 11:49:17 -0700 Subject: [PATCH 11/28] Include tillage page in Running Special Cases. --- doc/source/users_guide/running-special-cases/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/users_guide/running-special-cases/index.rst b/doc/source/users_guide/running-special-cases/index.rst index 7ead8f1551..5b6a6dea92 100644 --- a/doc/source/users_guide/running-special-cases/index.rst +++ b/doc/source/users_guide/running-special-cases/index.rst @@ -18,6 +18,7 @@ Running Special Cases running-the-prognostic-crop-model.rst running-with-irrigation.rst running-with-custom-crop-calendars.rst + running-with-tillage.rst Spinning-up-the-Satellite-Phenology-Model-CLMSP-spinup.rst Spinning-up-the-biogeochemistry-BGC-spinup.rst Running-with-excess-ground-ice.rst From 76e2d704eb0cbba5fece28e9095409e3a1157475 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Mon, 26 Feb 2024 11:49:34 -0700 Subject: [PATCH 12/28] Update Running With Tillage to reflect default "low" setting. --- .../running-special-cases/Running-with-tillage.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/source/users_guide/running-special-cases/Running-with-tillage.rst b/doc/source/users_guide/running-special-cases/Running-with-tillage.rst index 77309aea07..8cfcaa680b 100644 --- a/doc/source/users_guide/running-special-cases/Running-with-tillage.rst +++ b/doc/source/users_guide/running-special-cases/Running-with-tillage.rst @@ -7,25 +7,25 @@ ===================== -Cropland tillage (Sect. :numref:`decomp_mgmt_modifiers`) can be toggled by specifying a value of ``'low'`` (low intensity) or ``'high'`` (high intensity) for the ``tillage_mode`` namelist option. By default this option is ``'off'``. +Cropland tillage (Sect. :numref:`decomp_mgmt_modifiers`) is set to ``'low'`` by default. This can be changed to a value of ``'off'`` (no tillage) or ``'high'`` (high-intensity tillage) for the ``tillage_mode`` namelist option. Depth of tillage can be changed with the ``max_tillage_depth`` parameter (meters; default 0.26). Tillage multipliers for different soil pools and time since planting are defined on the parameter file, in variables ``bgc_till_decompk_multipliers`` (for CENTURY soil) and ``mimics_till_decompk_multipliers`` (for MIMICS soil). These variables were originally added with the script at ``tools/contrib/add_tillage_to_paramsfile.py``, which can be modified as needed to change tillage multipliers. -Example: Crop simulation with tillage -------------------------------------- +Example: Crop simulation with no tillage +---------------------------------------- :: - > cime/scripts/create_newcase -case IHistClm51BgcCrop_till -res f19_g17_gl4 -compset IHistClm51BgcCrop + > cime/scripts/create_newcase -case IHistClm51BgcCrop_notill -res f19_g17_gl4 -compset IHistClm51BgcCrop - > cd IHistClm51BgcCrop_till + > cd IHistClm51BgcCrop_notill > ./case.setup - # turn on tillage ('low' or 'high'; default 'off') - > echo "tillage_mode = 'high'" >> user_nl_clm + # turn off tillage + > echo "tillage_mode = 'off'" >> user_nl_clm Reverting fixes relative to original tillage implementation ----------------------------------------------------------- From 6b66e68c84e3a521f63cde3e42e1b2b0ad06ea7a Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Wed, 28 Feb 2024 22:44:04 -0700 Subject: [PATCH 13/28] Run 'git lfs install' in 'make fetch-images'. Should have no effect if it's already initialized, except some extra verbosity. --- doc/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/Makefile b/doc/Makefile index 1b8a86ad9a..49e9764b7a 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -17,6 +17,7 @@ help: # have configured this repository (via an .lfsconfig file at the top level) to NOT # automatically fetch any of the large files when cloning / fetching. fetch-images: + git lfs install git lfs pull --exclude="" --include="" .PHONY: help fetch-images Makefile From 14ad3e3952f35c2352c9d65ada23f353b633b428 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Thu, 29 Feb 2024 09:55:02 -0700 Subject: [PATCH 14/28] Fix filenames in running-special-cases/. --- ...model.rst => Running-the-prognostic-crop-model.rst} | 0 ...dars.rst => Running-with-custom-crop-calendars.rst} | 0 ...with-irrigation.rst => Running-with-irrigation.rst} | 0 ...s-a-special-case.rst => What-is-a-special-case.rst} | 0 doc/source/users_guide/running-special-cases/index.rst | 10 +++++----- 5 files changed, 5 insertions(+), 5 deletions(-) rename doc/source/users_guide/running-special-cases/{running-the-prognostic-crop-model.rst => Running-the-prognostic-crop-model.rst} (100%) rename doc/source/users_guide/running-special-cases/{running-with-custom-crop-calendars.rst => Running-with-custom-crop-calendars.rst} (100%) rename doc/source/users_guide/running-special-cases/{running-with-irrigation.rst => Running-with-irrigation.rst} (100%) rename doc/source/users_guide/running-special-cases/{what-is-a-special-case.rst => What-is-a-special-case.rst} (100%) diff --git a/doc/source/users_guide/running-special-cases/running-the-prognostic-crop-model.rst b/doc/source/users_guide/running-special-cases/Running-the-prognostic-crop-model.rst similarity index 100% rename from doc/source/users_guide/running-special-cases/running-the-prognostic-crop-model.rst rename to doc/source/users_guide/running-special-cases/Running-the-prognostic-crop-model.rst diff --git a/doc/source/users_guide/running-special-cases/running-with-custom-crop-calendars.rst b/doc/source/users_guide/running-special-cases/Running-with-custom-crop-calendars.rst similarity index 100% rename from doc/source/users_guide/running-special-cases/running-with-custom-crop-calendars.rst rename to doc/source/users_guide/running-special-cases/Running-with-custom-crop-calendars.rst diff --git a/doc/source/users_guide/running-special-cases/running-with-irrigation.rst b/doc/source/users_guide/running-special-cases/Running-with-irrigation.rst similarity index 100% rename from doc/source/users_guide/running-special-cases/running-with-irrigation.rst rename to doc/source/users_guide/running-special-cases/Running-with-irrigation.rst diff --git a/doc/source/users_guide/running-special-cases/what-is-a-special-case.rst b/doc/source/users_guide/running-special-cases/What-is-a-special-case.rst similarity index 100% rename from doc/source/users_guide/running-special-cases/what-is-a-special-case.rst rename to doc/source/users_guide/running-special-cases/What-is-a-special-case.rst diff --git a/doc/source/users_guide/running-special-cases/index.rst b/doc/source/users_guide/running-special-cases/index.rst index 5b6a6dea92..9173825d04 100644 --- a/doc/source/users_guide/running-special-cases/index.rst +++ b/doc/source/users_guide/running-special-cases/index.rst @@ -14,11 +14,11 @@ Running Special Cases .. toctree:: :maxdepth: 2 - what-is-a-special-case.rst - running-the-prognostic-crop-model.rst - running-with-irrigation.rst - running-with-custom-crop-calendars.rst - running-with-tillage.rst + What-is-a-special-case.rst + Running-the-prognostic-crop-model.rst + Running-with-irrigation.rst + Running-with-custom-crop-calendars.rst + Running-with-tillage.rst Spinning-up-the-Satellite-Phenology-Model-CLMSP-spinup.rst Spinning-up-the-biogeochemistry-BGC-spinup.rst Running-with-excess-ground-ice.rst From c858aed302e5967e9444d6c607e3058706807156 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 5 Mar 2024 10:24:04 -0700 Subject: [PATCH 15/28] Remove duplicates of mpi-serial DEBUG tests for Derecho with intel AND gnu, use one or the other, roughly alturnate between them --- cime_config/testdefs/testlist_clm.xml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/cime_config/testdefs/testlist_clm.xml b/cime_config/testdefs/testlist_clm.xml index 0c8bd9a4ff..e1a0f45b1a 100644 --- a/cime_config/testdefs/testlist_clm.xml +++ b/cime_config/testdefs/testlist_clm.xml @@ -1986,8 +1986,6 @@ - - @@ -2106,7 +2104,6 @@ - @@ -2120,7 +2117,6 @@ - @@ -2131,7 +2127,6 @@ - @@ -2168,8 +2163,6 @@ - - @@ -2183,8 +2176,6 @@ - - @@ -2197,10 +2188,7 @@ - - - @@ -2592,8 +2580,6 @@ - - @@ -2614,7 +2600,6 @@ - @@ -2949,7 +2934,6 @@ - @@ -2977,7 +2961,6 @@ - From 0dd117d60df377eebc49bee1cb345e060a8a0b63 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Sat, 2 Mar 2024 13:44:23 -0700 Subject: [PATCH 16/28] Tech Note: Update and simplify text in Harvest section. * Correction: 50% of residues are now removed, not 0. * Removes huge table that can be summarized as "0.7 for biofuel crops and 0 for everything else." --- .../CLM50_Tech_Note_Crop_Irrigation.rst | 68 +++---------------- ...ech_Note_Vegetation_Phenology_Turnover.rst | 2 +- 2 files changed, 9 insertions(+), 61 deletions(-) diff --git a/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst b/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst index 42238c4273..5b123b7565 100755 --- a/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst +++ b/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst @@ -509,9 +509,15 @@ where :math:`{C}_{leaf}`, :math:`{C}_{stem}`, and :math:`{C}_{froot}` is the car Harvest ''''''' -Variables track the flow of grain C and N to food and of all other plant pools, including live stem C and N, to litter, and to biofuel feedstock. A fraction (determined by the :math:`biofuel\_harvfrac`, defined in :numref:`Table Plant functional type (PFT) parameters for harvested fraction of leaf/livestem for bioenergy production`) of leaf/livestem C and N from bioenergy crops is removed at harvest for biofuels (Equations :eq:`25.9`. :eq:`harv_c_to_removed_residue`, :eq:`25.12`, and :eq:`harv_n_to_removed_residue`), with the remaining portions going to the litter pools (Equations :eq:`20.14)`, :eq:`25.11`, and :eq:`25.14`). Putting live stem C and N into the litter and biofuel pools is in contrast to the approach for unmanaged PFTs which puts live stem C and N into dead stem pools first. Biofuel crop leaf and stem C and N pools are routed to the litter and biofuel pools, in contrast to that of unmanaged PFTs and non-biofuel crops, which under default settings put leaf C and N into litter pools only. All crops can have their leaf and stem pools routed to a "removed residue" pool by setting namelist parameter :math:`crop\_residue\_removal\_frac` to something greater than its default zero. Root C and N pools are routed to the litter pools in the same manner as natural vegetation. +Whereas live crop C and N in grain was formerly transferred to the litter pool upon harvest, CLM5 splits this between "food" and "seed" pools. In the former—more generally a "crop product" pool—C and N decay to the atmosphere over one year, similar to how the wood product pools work. The latter is used in the subsequent year to account for the C and N required for crop seeding. -In the equations below, subscript :math:`p` refers to either the leaf or live stem biomass pool. +Live leaf and stem biomass at harvest is transferred to biofuel, removed residue, and/or litter pools. + +For the biofuel crops Miscanthus and switchgrass, 70% of live leaf and stem biomass at harvest is transferred to the crop product pool as described for "food" harvest above. This value can be changed for these crops—or set to something other than the default zero for any other crop—with the parameter :math:`biofuel\_harvfrac`. + +50% of any remaining live leaf and stem biomass at harvest (after biofuel removal, if any) is removed to the crop product pool to represent off-field uses such as use for animal feed and bedding. This value can be changed with the parameter :math:`crop\_residue\_removal\_frac` (0–1). + +The following equations illustrate how this works. Subscript :math:`p` refers to either the leaf or live stem biomass pool. .. math:: :label: 25.9 @@ -553,64 +559,6 @@ with corresponding nitrogen fluxes: where CF is the carbon flux, CS is stored carbon, NF is the nitrogen flux, NS is stored nitrogen, and :math:`biofuel\_harvfrac` is the harvested fraction of leaf/livestem for biofuel feedstocks. -.. _Table Plant functional type (PFT) parameters for harvested fraction of leaf/livestem for bioenergy production: - -.. table:: Plant functional type (PFT) parameters for harvested fraction of leaf/livestem for bioenergy production. - - +----------------------------------+----------------------------+ - | PFT | :math:`biofuel\_harvfrac` | - +==================================+============================+ - | NET Temperate | 0.00 | - +----------------------------------+----------------------------+ - | NET Boreal | 0.00 | - +----------------------------------+----------------------------+ - | NDT Boreal | 0.00 | - +----------------------------------+----------------------------+ - | BET Tropical | 0.00 | - +----------------------------------+----------------------------+ - | BET temperate | 0.00 | - +----------------------------------+----------------------------+ - | BDT tropical | 0.00 | - +----------------------------------+----------------------------+ - | BDT temperate | 0.00 | - +----------------------------------+----------------------------+ - | BDT boreal | 0.00 | - +----------------------------------+----------------------------+ - | BES temperate | 0.00 | - +----------------------------------+----------------------------+ - | BDS temperate | 0.00 | - +----------------------------------+----------------------------+ - | BDS boreal | 0.00 | - +----------------------------------+----------------------------+ - | C\ :sub:`3` arctic grass | 0.00 | - +----------------------------------+----------------------------+ - | C\ :sub:`3` grass | 0.00 | - +----------------------------------+----------------------------+ - | C\ :sub:`4` grass | 0.00 | - +----------------------------------+----------------------------+ - | Temperate Corn | 0.00 | - +----------------------------------+----------------------------+ - | Spring Wheat | 0.00 | - +----------------------------------+----------------------------+ - | Temperate Soybean | 0.00 | - +----------------------------------+----------------------------+ - | Cotton | 0.00 | - +----------------------------------+----------------------------+ - | Rice | 0.00 | - +----------------------------------+----------------------------+ - | Sugarcane | 0.00 | - +----------------------------------+----------------------------+ - | Tropical Corn | 0.00 | - +----------------------------------+----------------------------+ - | Tropical Soybean | 0.00 | - +----------------------------------+----------------------------+ - | Miscanthus | 0.70 | - +----------------------------------+----------------------------+ - | Switchgrass | 0.70 | - +----------------------------------+----------------------------+ - -Whereas food C and N was formerly transferred to the litter pool, CLM5 routes food C and N to a grain product pool where the C and N decay to the atmosphere over one year, similar in structure to the wood product pools. Biofuel and removed-residue C and N is also routed to the grain product pool and decays to the atmosphere over one year. Additionally, CLM5 accounts for the C and N required for crop seeding by removing the seed C and N from the grain product pool during harvest. The crop seed pool is then used to seed crops in the subsequent year. - Annual food crop yields (g dry matter m\ :sup:`-2`) can be calculated by saving the GRAINC_TO_FOOD_ANN variable once per year, then postprocessing with Equation :eq:`25.15`. This calculation assumes that grain C is 45% of the total dry weight. Additionally, harvest is not typically 100% efficient, so analysis needs to assume that harvest efficiency is less---we use 85%. .. math:: diff --git a/doc/source/tech_note/Vegetation_Phenology_Turnover/CLM50_Tech_Note_Vegetation_Phenology_Turnover.rst b/doc/source/tech_note/Vegetation_Phenology_Turnover/CLM50_Tech_Note_Vegetation_Phenology_Turnover.rst index 1665b9a00f..5bb4dc9e40 100644 --- a/doc/source/tech_note/Vegetation_Phenology_Turnover/CLM50_Tech_Note_Vegetation_Phenology_Turnover.rst +++ b/doc/source/tech_note/Vegetation_Phenology_Turnover/CLM50_Tech_Note_Vegetation_Phenology_Turnover.rst @@ -118,7 +118,7 @@ The deciduous phenology algorithms also specify the occurrence of litterfall dur r_{xfer\_ off} =\frac{2\Delta t}{t_{offset} ^{2} } -where superscripts *n* and *n-1* refer to fluxes on the current and previous timesteps, respectively. The rate coefficient :math:`{r}_{xfer\_off}` varies with time to produce a linearly increasing litterfall rate throughout the offset period. The :math:`biofuel\_harvfrac` (:numref:`Table Plant functional type (PFT) parameters for harvested fraction of leaf/livestem for bioenergy production`) is the harvested fraction of aboveground biomass (leaf & livestem) for bioenergy crops. The special case for fluxes in the final litterfall timestep (:math:`{t}_{offset}` = :math:`\Delta t`\ ) ensures that all of the displayed growth is sent to the litter pools or biofuel feedstock pools. The fraction (:math:`biofuel\_harvfrac`) of leaf biomass going to the biofuel feedstock pools (Equation :eq:`25.9`) is defined in Table 26.3 and is only non-zero for prognostic crops. The remaining fraction of leaf biomass (:math:`1-biofuel\_harvfrac`) for deciduous plant types is sent to the litter pools. Similar modifications made for livestem carbon pools for prognostic crops can be found in section :numref:`Harvest to food and seed` in Equations :eq:`25.9`-:eq:`25.14`. +where superscripts *n* and *n-1* refer to fluxes on the current and previous timesteps, respectively. The rate coefficient :math:`{r}_{xfer\_off}` varies with time to produce a linearly increasing litterfall rate throughout the offset period. The :math:`biofuel\_harvfrac` (:numref:`Harvest to food and seed`) is the harvested fraction of aboveground biomass (leaf & livestem) for bioenergy crops. The special case for fluxes in the final litterfall timestep (:math:`{t}_{offset}` = :math:`\Delta t`\ ) ensures that all of the displayed growth is sent to the litter pools or biofuel feedstock pools. The fraction (:math:`biofuel\_harvfrac`) of leaf biomass going to the biofuel feedstock pools (Equation :eq:`25.9`) is defined in Table 26.3 and is only non-zero for prognostic crops. The remaining fraction of leaf biomass (:math:`1-biofuel\_harvfrac`) for deciduous plant types is sent to the litter pools. Similar modifications made for livestem carbon pools for prognostic crops can be found in section :numref:`Harvest to food and seed` in Equations :eq:`25.9`-:eq:`25.14`. Corresponding nitrogen fluxes during litterfall take into account retranslocation of nitrogen out of the displayed leaf pool prior to litterfall (:math:`{NF}_{leaf,retrans}`, gN m\ :sup:`-2` s\ :sup:`-1`). Retranslocation of nitrogen out of fine roots is assumed to be negligible. The fluxes are: From bf2244d04b369cec4e2e1efa33de31d3b31c74c1 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Sat, 2 Mar 2024 13:44:40 -0700 Subject: [PATCH 17/28] Explain reasoning behind default 50% residue removal. --- .../Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst | 2 +- .../tech_note/References/CLM50_Tech_Note_References.rst | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst b/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst index 5b123b7565..e29abcaf03 100755 --- a/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst +++ b/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst @@ -515,7 +515,7 @@ Live leaf and stem biomass at harvest is transferred to biofuel, removed residue For the biofuel crops Miscanthus and switchgrass, 70% of live leaf and stem biomass at harvest is transferred to the crop product pool as described for "food" harvest above. This value can be changed for these crops—or set to something other than the default zero for any other crop—with the parameter :math:`biofuel\_harvfrac`. -50% of any remaining live leaf and stem biomass at harvest (after biofuel removal, if any) is removed to the crop product pool to represent off-field uses such as use for animal feed and bedding. This value can be changed with the parameter :math:`crop\_residue\_removal\_frac` (0–1). +50% of any remaining live leaf and stem biomass at harvest (after biofuel removal, if any) is removed to the crop product pool to represent off-field uses such as use for animal feed and bedding. This value can be changed with the parameter :math:`crop\_residue\_removal\_frac` (0–1). The default 50% is derived from :ref:`Smerald et al. 2023 `, who found a global average of 50% of residues left on the field. This includes residues burned in the field, meaning that our implementation implictly assumes the CLM crop burning representation will handle those residues appropriately. The following equations illustrate how this works. Subscript :math:`p` refers to either the leaf or live stem biomass pool. diff --git a/doc/source/tech_note/References/CLM50_Tech_Note_References.rst b/doc/source/tech_note/References/CLM50_Tech_Note_References.rst index b824f705bd..eafd44e8f4 100644 --- a/doc/source/tech_note/References/CLM50_Tech_Note_References.rst +++ b/doc/source/tech_note/References/CLM50_Tech_Note_References.rst @@ -1264,6 +1264,10 @@ Sitch, S et al. (2003). Evaluation of ecosystem dynamics, plant geography and te Sivak, M. 2013. Air conditioning versus heating: climate control is more energy demanding in Minneapolis than in Miami. Environ. Res. Lett., 8, doi:10.1088/1748-9326/8/1/014050. +.. _Smeraldetal2023: + +Smerald, A., Rahimi, J., & Scheer, C., 2023. A global dataset for the production and usage of cereal residues in the period 1997–2021. Scientific Data, 10(1), 685. doi: 10.1038/s41597-023-02587-0 + .. _smith2001: Smith, B., I.C. Prentice, and M.T. Sykes, 2001. Representation of vegetation dynamics in the modelling of terrestrial ecosystems: comparing two contrasting approaches within European climate space. Global Ecology and Biogeography 10.6, pp. 621-637. From 96369ddffa970b1d16f3fb561c6805d5e13e61dd Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Sat, 2 Mar 2024 13:46:29 -0700 Subject: [PATCH 18/28] Clarify units of biofuel_harvfrac. --- .../Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst b/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst index e29abcaf03..84021ddcb2 100755 --- a/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst +++ b/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst @@ -513,7 +513,7 @@ Whereas live crop C and N in grain was formerly transferred to the litter pool u Live leaf and stem biomass at harvest is transferred to biofuel, removed residue, and/or litter pools. -For the biofuel crops Miscanthus and switchgrass, 70% of live leaf and stem biomass at harvest is transferred to the crop product pool as described for "food" harvest above. This value can be changed for these crops—or set to something other than the default zero for any other crop—with the parameter :math:`biofuel\_harvfrac`. +For the biofuel crops Miscanthus and switchgrass, 70% of live leaf and stem biomass at harvest is transferred to the crop product pool as described for "food" harvest above. This value can be changed for these crops—or set to something other than the default zero for any other crop—with the parameter :math:`biofuel\_harvfrac` (0-1). 50% of any remaining live leaf and stem biomass at harvest (after biofuel removal, if any) is removed to the crop product pool to represent off-field uses such as use for animal feed and bedding. This value can be changed with the parameter :math:`crop\_residue\_removal\_frac` (0–1). The default 50% is derived from :ref:`Smerald et al. 2023 `, who found a global average of 50% of residues left on the field. This includes residues burned in the field, meaning that our implementation implictly assumes the CLM crop burning representation will handle those residues appropriately. From ac6a4157ad76d03d5d57827d8cab8c1b7b131a65 Mon Sep 17 00:00:00 2001 From: Sam Rabin Date: Sat, 2 Mar 2024 13:50:50 -0700 Subject: [PATCH 19/28] Mention residue removal in "changes since CLM5." --- .../Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst b/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst index 84021ddcb2..2840d37176 100755 --- a/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst +++ b/doc/source/tech_note/Crop_Irrigation/CLM50_Tech_Note_Crop_Irrigation.rst @@ -41,6 +41,7 @@ Available new features since the CLM5 release - Addition of bioenergy crops - Ability to customize crop calendars (sowing windows/dates, maturity requirements) using stream files - Cropland soil tillage +- Crop residue removal .. _The crop model: From 16049e6842a1da8bf0f4926bb8e5c5a7aeaee777 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 8 Mar 2024 12:42:03 -0700 Subject: [PATCH 20/28] Fix the number of tests --- bld/unit_testers/build-namelist_test.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index 58b3056ef8..ceaa0d14d0 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -163,7 +163,7 @@ sub cat_and_create_namelistinfile { # # Figure out number of tests that will run # -my $ntests = 1999; +my $ntests = 1996; if ( defined($opts{'compare'}) ) { $ntests += 1353; From 99d54241c7df4f8e0d6ba6875a5ca018f56a6348 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Fri, 8 Mar 2024 17:07:45 -0700 Subject: [PATCH 21/28] Turn off the user_nl_clm_real_parameter files and testing for it as not really needed this resolves #2412 --- bld/unit_testers/build-namelist_test.pl | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/bld/unit_testers/build-namelist_test.pl b/bld/unit_testers/build-namelist_test.pl index ceaa0d14d0..6224acc815 100755 --- a/bld/unit_testers/build-namelist_test.pl +++ b/bld/unit_testers/build-namelist_test.pl @@ -163,10 +163,10 @@ sub cat_and_create_namelistinfile { # # Figure out number of tests that will run # -my $ntests = 1996; +my $ntests = 1549; if ( defined($opts{'compare'}) ) { - $ntests += 1353; + $ntests += 907; } plan( tests=>$ntests ); @@ -190,8 +190,7 @@ sub cat_and_create_namelistinfile { &make_config_cache($phys); my $DOMFILE = "$inputdata_rootdir/atm/datm7/domain.lnd.T31_gx3v7.090928.nc"; -my $real_par_file = "user_nl_ctsm_real_parameters"; -my $bldnml = "../build-namelist -verbose -csmdata $inputdata_rootdir -configuration clm -structure standard -glc_nec 10 -no-note -output_reals $real_par_file"; +my $bldnml = "../build-namelist -verbose -csmdata $inputdata_rootdir -configuration clm -structure standard -glc_nec 10 -no-note"; if ( $opts{'test'} ) { $bldnml .= " -test"; } @@ -201,7 +200,7 @@ sub cat_and_create_namelistinfile { system( "/bin/rm $tempfile" ); } -my @files = ( "lnd_in", $tempfile, $real_par_file ); +my @files = ( "lnd_in", $tempfile ); my $cwd = `pwd`; chomp( $cwd ); my $cfiles = NMLTest::CompFiles->new( $cwd, @files ); @@ -269,13 +268,11 @@ sub cat_and_create_namelistinfile { $cfiles->copyfiles( "most_options", $mode ); # Compare to default $cfiles->doNOTdodiffonfile( "lnd_in", "default", $mode ); - $cfiles->doNOTdodiffonfile( "$real_par_file", "default", $mode ); $cfiles->doNOTdodiffonfile( "$tempfile", "default", $mode ); $cfiles->comparefiles( "default", $mode ); # Compare to baseline if ( defined($opts{'compare'}) ) { $cfiles->dodiffonfile( "lnd_in", "most_options", $mode ); - $cfiles->dodiffonfile( "$real_par_file", "most_options", $mode ); $cfiles->doNOTdodiffonfile( "$tempfile", "most_options", $mode ); $cfiles->comparefiles( "most_options", $mode, $opts{'compare'} ); } @@ -358,7 +355,6 @@ sub cat_and_create_namelistinfile { } if ( defined($opts{'compare'}) ) { $cfiles->doNOTdodiffonfile( "$tempfile", "$base_options $options", $mode ); - $cfiles->dodiffonfile( "$real_par_file", "$base_options $options", $mode ); $cfiles->comparefiles( "$base_options $options", $mode, $opts{'compare'} ); } if ( defined($opts{'generate'}) ) { @@ -408,7 +404,6 @@ sub cat_and_create_namelistinfile { if ( defined($opts{'compare'}) ) { $cfiles->doNOTdodiffonfile( "$tempfile", "$options", $mode ); $cfiles->dodiffonfile( "lnd_in", "$options", $mode ); - $cfiles->dodiffonfile( "$real_par_file", "$options", $mode ); $cfiles->comparefiles( "$options", $mode, $opts{'compare'} ); } if ( defined($opts{'generate'}) ) { @@ -445,7 +440,6 @@ sub cat_and_create_namelistinfile { if ( defined($opts{'compare'}) ) { $cfiles->doNOTdodiffonfile( "$tempfile", "$options", $mode ); $cfiles->dodiffonfile( "lnd_in", "$options", $mode ); - $cfiles->dodiffonfile( "$real_par_file", "$options", $mode ); $cfiles->comparefiles( "$options", $mode, $opts{'compare'} ); } if ( defined($opts{'generate'}) ) { @@ -486,7 +480,6 @@ sub cat_and_create_namelistinfile { if ( defined($opts{'compare'}) ) { $cfiles->doNOTdodiffonfile( "$tempfile", "$options", $mode ); $cfiles->dodiffonfile( "lnd_in", "$options", $mode ); - $cfiles->dodiffonfile( "$real_par_file", "$options", $mode ); $cfiles->comparefiles( "$options", $mode, $opts{'compare'} ); } if ( defined($opts{'generate'}) ) { @@ -1388,7 +1381,6 @@ sub cat_and_create_namelistinfile { $cfiles->shownmldiff( "default", "standard" ); if ( defined($opts{'compare'}) ) { $cfiles->doNOTdodiffonfile( "$tempfile", "$options", $mode ); - $cfiles->dodiffonfile( "$real_par_file", "$options", $mode ); $cfiles->comparefiles( "$options", $mode, $opts{'compare'} ); } @@ -1473,7 +1465,6 @@ sub cat_and_create_namelistinfile { $cfiles->shownmldiff( "default", "standard" ); if ( defined($opts{'compare'}) ) { $cfiles->doNOTdodiffonfile( "$tempfile", "$options", $mode ); - $cfiles->dodiffonfile( "$real_par_file", "$options", $mode ); $cfiles->comparefiles( "$options", $mode, $opts{'compare'} ); } if ( defined($opts{'generate'}) ) { @@ -1547,7 +1538,6 @@ sub cat_and_create_namelistinfile { $cfiles->shownmldiff( "default", "standard" ); if ( defined($opts{'compare'}) ) { $cfiles->doNOTdodiffonfile( "$tempfile", "$options", $mode ); - $cfiles->dodiffonfile( "$real_par_file", "$options", $mode ); $cfiles->comparefiles( "$options", $mode, $opts{'compare'} ); } if ( defined($opts{'generate'}) ) { @@ -1574,7 +1564,6 @@ sub cat_and_create_namelistinfile { $cfiles->shownmldiff( "default", "standard" ); if ( defined($opts{'compare'}) ) { $cfiles->doNOTdodiffonfile( "$tempfile", "$options", $mode ); - $cfiles->dodiffonfile( "$real_par_file", "$options", $mode ); $cfiles->comparefiles( "$options", $mode, $opts{'compare'} ); } if ( defined($opts{'generate'}) ) { @@ -1757,7 +1746,6 @@ sub cleanup { my $type = shift; print "Cleanup files created\n"; - system( "/bin/rm env_run.xml $real_par_file" ); if ( defined($type) ) { if ( $type eq "config" ) { system( "/bin/rm config_cache.xml" ); From 6812720d7787488d1048a68d2a4ae75f13fe8886 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 7 Mar 2024 00:25:47 -0700 Subject: [PATCH 22/28] Save the current directory at setup and return to it before deleting the temporary directory, this is handling for #2405 --- python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py | 2 ++ python/ctsm/test/test_sys_fsurdat_modifier.py | 2 ++ python/ctsm/test/test_sys_lilac_build_ctsm.py | 2 ++ python/ctsm/test/test_sys_modify_singlept_site_neon.py | 2 ++ python/ctsm/test/test_sys_regrid_ggcmi_shdates.py | 2 ++ python/ctsm/test/test_unit_fsurdat_modifier.py | 2 ++ python/ctsm/test/test_unit_modify_singlept_site_neon.py | 2 ++ python/ctsm/test/test_unit_neon_arg_parse.py | 2 ++ python/ctsm/test/test_unit_neon_site.py | 2 ++ python/ctsm/test/test_unit_path_utils.py | 2 ++ python/ctsm/test/test_unit_run_neon.py | 2 ++ python/ctsm/test/test_unit_utils.py | 2 ++ python/ctsm/test/test_unit_utils_import_coord.py | 2 ++ 13 files changed, 26 insertions(+) diff --git a/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py b/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py index 46b59f4e76..59610751bc 100755 --- a/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py +++ b/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py @@ -21,9 +21,11 @@ class TestJobLauncherNoBatch(unittest.TestCase): """Tests of job_launcher_no_batch""" def setUp(self): + self._previous_dir = os.getcwd() self._testdir = tempfile.mkdtemp() def tearDown(self): + os.chdir(self._previous_dir ) shutil.rmtree(self._testdir, ignore_errors=True) def assertFileContentsEqual(self, expected, filepath, msg=None): diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 1a5045c14d..1c8a8a0e04 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -38,6 +38,7 @@ def setUp(self): - modify_fsurdat.cfg - fsurdat_out.nc """ + self._previous_dir = os.getcwd() self._cfg_template_path = os.path.join( path_to_ctsm_root(), "tools/modify_input_files/modify_fsurdat_template.cfg" ) @@ -55,6 +56,7 @@ def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_no_files_given_fail(self): diff --git a/python/ctsm/test/test_sys_lilac_build_ctsm.py b/python/ctsm/test/test_sys_lilac_build_ctsm.py index d773749bf7..3d17b24427 100755 --- a/python/ctsm/test/test_sys_lilac_build_ctsm.py +++ b/python/ctsm/test/test_sys_lilac_build_ctsm.py @@ -26,6 +26,7 @@ class TestSysBuildCtsm(unittest.TestCase): """System tests for lilac_build_ctsm""" def setUp(self): + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() self.assertTrue(os.path.isdir(self._tempdir)) @@ -42,6 +43,7 @@ def setUp(self): self._ncarhost = None def tearDown(self): + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) if self._ncarhost is not None: os.environ["NCAR_HOST"] = self._ncarhost diff --git a/python/ctsm/test/test_sys_modify_singlept_site_neon.py b/python/ctsm/test/test_sys_modify_singlept_site_neon.py index 74362be4cd..036e8f39a4 100755 --- a/python/ctsm/test/test_sys_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_sys_modify_singlept_site_neon.py @@ -27,6 +27,7 @@ def setUp(self): Make /_tempdir for use by these tests. Check tempdir for history files """ + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() testinputs_path = os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs") self._cfg_file_path = os.path.join( @@ -37,6 +38,7 @@ def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_modify_site(self): diff --git a/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py b/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py index 6c2e230481..e2b304070d 100755 --- a/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py +++ b/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py @@ -38,6 +38,7 @@ def setUp(self): self._testinputs_cc_path = testinputs_cc_path # Make /_tempdir for use by these tests. + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() # Obtain path for the directory being created in /_tempdir @@ -73,6 +74,7 @@ def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_regrid_ggcmi_shdates(self): diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 166924903b..25fcc19fa2 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -46,6 +46,7 @@ def setUp(self): testinputs_path, "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc", ) + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() self._fsurdat_in = os.path.join( testinputs_path, @@ -80,6 +81,7 @@ def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_subgrid_and_idealized_fails(self): diff --git a/python/ctsm/test/test_unit_modify_singlept_site_neon.py b/python/ctsm/test/test_unit_modify_singlept_site_neon.py index 3a9d7d424c..d74d7d6237 100755 --- a/python/ctsm/test/test_unit_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_unit_modify_singlept_site_neon.py @@ -43,12 +43,14 @@ def setUp(self): Make /_tempdir for use by these tests. Check tempdir for history files """ + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_get_neon(self): diff --git a/python/ctsm/test/test_unit_neon_arg_parse.py b/python/ctsm/test/test_unit_neon_arg_parse.py index 7bae337709..68de8d3c77 100755 --- a/python/ctsm/test/test_unit_neon_arg_parse.py +++ b/python/ctsm/test/test_unit_neon_arg_parse.py @@ -34,12 +34,14 @@ def setUp(self): """ Make /_tempdir for use by these tests. """ + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_function(self): diff --git a/python/ctsm/test/test_unit_neon_site.py b/python/ctsm/test/test_unit_neon_site.py index 4828718272..cf10b7c132 100755 --- a/python/ctsm/test/test_unit_neon_site.py +++ b/python/ctsm/test/test_unit_neon_site.py @@ -33,12 +33,14 @@ def setUp(self): """ Make /_tempdir for use by these tests. """ + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_modify_user_nl_transient(self): diff --git a/python/ctsm/test/test_unit_path_utils.py b/python/ctsm/test/test_unit_path_utils.py index 566c458acc..e4905b18b2 100755 --- a/python/ctsm/test/test_unit_path_utils.py +++ b/python/ctsm/test/test_unit_path_utils.py @@ -22,9 +22,11 @@ class TestPathUtils(unittest.TestCase): """Tests of path_utils""" def setUp(self): + self._previous_dir = os.getcwd() self._testdir = tempfile.mkdtemp() def tearDown(self): + os.chdir(self._previous_dir ) shutil.rmtree(self._testdir, ignore_errors=True) def _ctsm_path_in_cesm(self): diff --git a/python/ctsm/test/test_unit_run_neon.py b/python/ctsm/test/test_unit_run_neon.py index a35608e249..9a5bac457d 100755 --- a/python/ctsm/test/test_unit_run_neon.py +++ b/python/ctsm/test/test_unit_run_neon.py @@ -32,12 +32,14 @@ def setUp(self): """ Make /_tempdir for use by these tests. """ + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_check_neon_listing(self): diff --git a/python/ctsm/test/test_unit_utils.py b/python/ctsm/test/test_unit_utils.py index a78928abcc..fe63e01883 100755 --- a/python/ctsm/test/test_unit_utils.py +++ b/python/ctsm/test/test_unit_utils.py @@ -21,9 +21,11 @@ class TestUtilsFillTemplateFile(unittest.TestCase): """Tests of utils: fill_template_file""" def setUp(self): + self._previous_dir = os.getcwd() self._testdir = tempfile.mkdtemp() def tearDown(self): + os.chdir(self._previous_dir ) shutil.rmtree(self._testdir, ignore_errors=True) def test_fillTemplateFile_basic(self): diff --git a/python/ctsm/test/test_unit_utils_import_coord.py b/python/ctsm/test/test_unit_utils_import_coord.py index 6e339a913f..6e62e72966 100755 --- a/python/ctsm/test/test_unit_utils_import_coord.py +++ b/python/ctsm/test/test_unit_utils_import_coord.py @@ -39,6 +39,7 @@ class TestUtilsImportCoord(unittest.TestCase): def setUp(self): """Setup for trying out the methods""" + self._previous_dir = os.getcwd() testinputs_path = os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs") self._testinputs_path = testinputs_path self._tempdir = tempfile.mkdtemp() @@ -55,6 +56,7 @@ def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_importcoord1d(self): From 20e3e7a8b5002608ec07f2f435a4a7a3cea88013 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 12 Mar 2024 17:08:36 -0600 Subject: [PATCH 23/28] Update to cime branch tag that fixes the LILAC python test problem --- Externals.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 35db4870d5..c09bd4041e 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -44,7 +44,7 @@ required = True local_path = cime protocol = git repo_url = https://github.com/ESMCI/cime -tag = cime6.0.217_httpsbranch02 +tag = cime6.0.217_httpsbranch03 required = True [cmeps] @@ -98,4 +98,4 @@ tag = v1.0.8 required = False [externals_description] -schema_version = 1.0.0 \ No newline at end of file +schema_version = 1.0.0 From 605addb4b94500be773c0fe4e89b5a1eab30c9c3 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 12 Mar 2024 22:06:20 -0600 Subject: [PATCH 24/28] Simple check for removing current directory resolving #2412 in a simple way --- python/ctsm/site_and_regional/neon_site.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index 31ae78f5ad..29b5d2c7b5 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -18,6 +18,7 @@ # pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order from ctsm import add_cime_to_path from ctsm.path_utils import path_to_ctsm_root +from ctsm.utils import abort from CIME import build from CIME.case import Case @@ -75,6 +76,9 @@ def build_base_case( if overwrite and os.path.isdir(case_path): print("Removing the existing case at: {}".format(case_path)) + if os.getcwd() == case_path: + abort("Trying to remove the directory tree that we are in") + shutil.rmtree(case_path) with Case(case_path, read_only=False) as case: @@ -211,6 +215,8 @@ def run_case( if os.path.isdir(case_root): if overwrite: print("---- removing the existing case -------") + if os.getcwd() == case_root: + abort("Trying to remove the directory tree that we are in") shutil.rmtree(case_root) elif rerun: with Case(case_root, read_only=False) as case: From 26481f2aa1db61d0471d018d78421e20e1327eef Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 12 Mar 2024 22:26:57 -0600 Subject: [PATCH 25/28] Add some extra checking --- python/ctsm/lilac_build_ctsm.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python/ctsm/lilac_build_ctsm.py b/python/ctsm/lilac_build_ctsm.py index b189cb56ea..e6132e618c 100644 --- a/python/ctsm/lilac_build_ctsm.py +++ b/python/ctsm/lilac_build_ctsm.py @@ -153,6 +153,14 @@ def build_ctsm( existing_inputdata = existing_machine or inputdata_path is not None _create_build_dir(build_dir=build_dir, existing_inputdata=existing_inputdata) + # Some error checking + if inputdata_path is not None: + if not os.path.isdir(inputdata_path): + abort( "Input inputdata_path directory does NOT exist = "+inputdata_path ) + + if not os.path.isdir(build_dir): + abort( "Input build_dir directory does NOT exist = "+build_dir ) + if machine is None: assert os_type is not None, "with machine absent, os_type must be given" assert netcdf_path is not None, "with machine absent, netcdf_path must be given" @@ -176,6 +184,7 @@ def build_ctsm( extra_fflags=extra_fflags, extra_cflags=extra_cflags, ) + assert os.path.isdir(cime_path), "cime_path must be a directory" _create_case( cime_path=cime_path, @@ -627,6 +636,7 @@ def _fill_out_machine_files( "CIME_OUTPUT_ROOT": build_dir, "GMAKE": gmake, "GMAKE_J": gmake_j, + "MAX_TASKS_PER_NODE": max_mpitasks_per_node, "MAX_MPITASKS_PER_NODE": max_mpitasks_per_node, "ESMF_MKFILE_PATH": esmf_mkfile_path, }, From 2500534eb0a83cc3aff94b30fb62e915054030bf Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 12 Mar 2024 22:28:11 -0600 Subject: [PATCH 26/28] Run through black --- python/ctsm/lilac_build_ctsm.py | 6 +++--- python/ctsm/site_and_regional/neon_site.py | 4 ++-- .../test/joblauncher/test_unit_job_launcher_no_batch.py | 2 +- python/ctsm/test/test_sys_fsurdat_modifier.py | 2 +- python/ctsm/test/test_sys_lilac_build_ctsm.py | 2 +- python/ctsm/test/test_sys_modify_singlept_site_neon.py | 2 +- python/ctsm/test/test_sys_regrid_ggcmi_shdates.py | 2 +- python/ctsm/test/test_unit_fsurdat_modifier.py | 2 +- python/ctsm/test/test_unit_modify_singlept_site_neon.py | 2 +- python/ctsm/test/test_unit_neon_arg_parse.py | 2 +- python/ctsm/test/test_unit_neon_site.py | 2 +- python/ctsm/test/test_unit_path_utils.py | 2 +- python/ctsm/test/test_unit_run_neon.py | 2 +- python/ctsm/test/test_unit_utils.py | 2 +- python/ctsm/test/test_unit_utils_import_coord.py | 2 +- 15 files changed, 18 insertions(+), 18 deletions(-) diff --git a/python/ctsm/lilac_build_ctsm.py b/python/ctsm/lilac_build_ctsm.py index e6132e618c..d7b92517c5 100644 --- a/python/ctsm/lilac_build_ctsm.py +++ b/python/ctsm/lilac_build_ctsm.py @@ -155,11 +155,11 @@ def build_ctsm( # Some error checking if inputdata_path is not None: - if not os.path.isdir(inputdata_path): - abort( "Input inputdata_path directory does NOT exist = "+inputdata_path ) + if not os.path.isdir(inputdata_path): + abort("Input inputdata_path directory does NOT exist = " + inputdata_path) if not os.path.isdir(build_dir): - abort( "Input build_dir directory does NOT exist = "+build_dir ) + abort("Input build_dir directory does NOT exist = " + build_dir) if machine is None: assert os_type is not None, "with machine absent, os_type must be given" diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index 29b5d2c7b5..9ad690becb 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -77,7 +77,7 @@ def build_base_case( if overwrite and os.path.isdir(case_path): print("Removing the existing case at: {}".format(case_path)) if os.getcwd() == case_path: - abort("Trying to remove the directory tree that we are in") + abort("Trying to remove the directory tree that we are in") shutil.rmtree(case_path) @@ -216,7 +216,7 @@ def run_case( if overwrite: print("---- removing the existing case -------") if os.getcwd() == case_root: - abort("Trying to remove the directory tree that we are in") + abort("Trying to remove the directory tree that we are in") shutil.rmtree(case_root) elif rerun: with Case(case_root, read_only=False) as case: diff --git a/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py b/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py index 59610751bc..4e22e3c085 100755 --- a/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py +++ b/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py @@ -25,7 +25,7 @@ def setUp(self): self._testdir = tempfile.mkdtemp() def tearDown(self): - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._testdir, ignore_errors=True) def assertFileContentsEqual(self, expected, filepath, msg=None): diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 1c8a8a0e04..0e11a204a8 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -56,7 +56,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_no_files_given_fail(self): diff --git a/python/ctsm/test/test_sys_lilac_build_ctsm.py b/python/ctsm/test/test_sys_lilac_build_ctsm.py index 3d17b24427..a8af9b2eab 100755 --- a/python/ctsm/test/test_sys_lilac_build_ctsm.py +++ b/python/ctsm/test/test_sys_lilac_build_ctsm.py @@ -43,7 +43,7 @@ def setUp(self): self._ncarhost = None def tearDown(self): - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) if self._ncarhost is not None: os.environ["NCAR_HOST"] = self._ncarhost diff --git a/python/ctsm/test/test_sys_modify_singlept_site_neon.py b/python/ctsm/test/test_sys_modify_singlept_site_neon.py index 036e8f39a4..76a78c3db5 100755 --- a/python/ctsm/test/test_sys_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_sys_modify_singlept_site_neon.py @@ -38,7 +38,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_modify_site(self): diff --git a/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py b/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py index e2b304070d..9209ebfe78 100755 --- a/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py +++ b/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py @@ -74,7 +74,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_regrid_ggcmi_shdates(self): diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 25fcc19fa2..bcb9099f61 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -81,7 +81,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_subgrid_and_idealized_fails(self): diff --git a/python/ctsm/test/test_unit_modify_singlept_site_neon.py b/python/ctsm/test/test_unit_modify_singlept_site_neon.py index d74d7d6237..db1fc1966d 100755 --- a/python/ctsm/test/test_unit_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_unit_modify_singlept_site_neon.py @@ -50,7 +50,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_get_neon(self): diff --git a/python/ctsm/test/test_unit_neon_arg_parse.py b/python/ctsm/test/test_unit_neon_arg_parse.py index 68de8d3c77..4a5b0b9e6c 100755 --- a/python/ctsm/test/test_unit_neon_arg_parse.py +++ b/python/ctsm/test/test_unit_neon_arg_parse.py @@ -41,7 +41,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_function(self): diff --git a/python/ctsm/test/test_unit_neon_site.py b/python/ctsm/test/test_unit_neon_site.py index cf10b7c132..8ef6034f94 100755 --- a/python/ctsm/test/test_unit_neon_site.py +++ b/python/ctsm/test/test_unit_neon_site.py @@ -40,7 +40,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_modify_user_nl_transient(self): diff --git a/python/ctsm/test/test_unit_path_utils.py b/python/ctsm/test/test_unit_path_utils.py index e4905b18b2..067809cbf6 100755 --- a/python/ctsm/test/test_unit_path_utils.py +++ b/python/ctsm/test/test_unit_path_utils.py @@ -26,7 +26,7 @@ def setUp(self): self._testdir = tempfile.mkdtemp() def tearDown(self): - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._testdir, ignore_errors=True) def _ctsm_path_in_cesm(self): diff --git a/python/ctsm/test/test_unit_run_neon.py b/python/ctsm/test/test_unit_run_neon.py index 9a5bac457d..904db885a9 100755 --- a/python/ctsm/test/test_unit_run_neon.py +++ b/python/ctsm/test/test_unit_run_neon.py @@ -39,7 +39,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_check_neon_listing(self): diff --git a/python/ctsm/test/test_unit_utils.py b/python/ctsm/test/test_unit_utils.py index fe63e01883..85ba2515dd 100755 --- a/python/ctsm/test/test_unit_utils.py +++ b/python/ctsm/test/test_unit_utils.py @@ -25,7 +25,7 @@ def setUp(self): self._testdir = tempfile.mkdtemp() def tearDown(self): - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._testdir, ignore_errors=True) def test_fillTemplateFile_basic(self): diff --git a/python/ctsm/test/test_unit_utils_import_coord.py b/python/ctsm/test/test_unit_utils_import_coord.py index 6e62e72966..11ff3c1b54 100755 --- a/python/ctsm/test/test_unit_utils_import_coord.py +++ b/python/ctsm/test/test_unit_utils_import_coord.py @@ -56,7 +56,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_importcoord1d(self): From 29c4fb95997787c156fb2b2c452f36f6253d92fb Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 12 Mar 2024 22:29:41 -0600 Subject: [PATCH 27/28] Add black commit to blame list --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 8708f8e0c2..58a1869f9d 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -30,3 +30,4 @@ d866510188d26d51bcd6d37239283db690af7e82 8a168bb0895f4f2421608dd2589398e13a6663e6 183fc26a6691bbdf87f515dc47924a64be3ced9b 6fccf682eaf718615407d9bacdd3903b8786a03d +2500534eb0a83cc3aff94b30fb62e915054030bf From 94795eda213a5a14c1d4667ad1d3f02ce9da592e Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Wed, 13 Mar 2024 00:33:18 -0600 Subject: [PATCH 28/28] Update Change files --- doc/ChangeLog | 111 ++++++++++++++++++++++++++++++++++++++++++++++++++ doc/ChangeSum | 1 + 2 files changed, 112 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 30058a80b4..9ef124937f 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,4 +1,115 @@ =============================================================== +Tag name: ctsm5.1.dev172 +Originator(s): erik (Erik Kluzek,UCAR/TSS,303-497-1326) +Date: Tue 12 Mar 2024 11:59:48 PM MDT +One-line Summary: Merge b4b-dev + +Purpose and description of changes +---------------------------------- + +Update of externals to what's expected in cesm2_3_beta17. Some documentation updates including a rebuild +of the documentation and updating how images are handled with git lfs. Improvements to the documentation for +residue removal and tillage for prognostic crops. + +Significant changes to scientifically-supported configurations +-------------------------------------------------------------- + +Does this tag change answers significantly for any of the following physics configurations? +(Details of any changes will be given in the "Answer changes" section below.) + + [Put an [X] in the box for any configuration with significant answer changes.] + +[ ] clm5_1 + +[ ] clm5_0 + +[ ] ctsm5_0-nwp + +[ ] clm4_5 + + +Bugs fixed +---------- + +CTSM issues fixed (include CTSM Issue #): + Fixes #2351 Update to cesm2_3_beta17 externals + Fixes #2380 ctsm_pylib on izumi can use python3.7.9 + Fixes #2331 py_env_create fails on izumi + Fixes #1910 modify_singlept_neon on izumi + Fixes #1658 malformed file on izumi + Fixes #2412 Check current directory isn't being removed in run_neon + +Notes of particular relevance for users +--------------------------------------- + +Notes of particular relevance for developers: +--------------------------------------------- +NOTE: Be sure to review the steps in README.CHECKLIST.master_tags as well as the coding style in the Developers Guide +[Remove any lines that don't apply. Remove entire section if nothing applies.] + +Caveats for developers (e.g., code that is duplicated that requires double maintenance): + + ccs_config config_machines.xml files were updated from v2 to v3 + + The timing and memory usage information was moved from drv.log to med.log for nuopc runs. + This allows the creating of the baselines cpl-mem.log and cpl-tput.log files. + + SMP_PRESENT removed and BUILD_THREADED added. + +Changes to tests or testing: + Bring back DEBUG testing with intel for mpi-serial cases. + + +Testing summary: regular +---------------- + [PASS means all tests PASS; OK means tests PASS other than expected fails.] + + build-namelist tests (if CLMBuildNamelist.pm has changed): + + derecho - PASS + + python testing (if python code has changed; see instructions in python/README.md; document testing done): + + derecho - PASS + + clm_pymods test suite on derecho - PASS + + regular tests (aux_clm: https://github.com/ESCOMP/CTSM/wiki/System-Testing-Guide#pre-merge-system-testing): + + derecho ----- OK + izumi ------- OK + +If the tag used for baseline comparisons was NOT the previous tag, note that here: + + +Answer changes +-------------- + +Changes answers relative to baseline: no bit-for-bit + +Other details +------------- +List any externals directories updated (cime, rtm, mosart, cism, fates, etc.): + manage externals + cismwrap_2_1_97 + = rtm1_0_79 + mosart1_0_49 + ccs_config_cesm0.0.92 + cime6.0.217_httpsbranch03 + cmeps0.14.50 + cdeps1.0.28 + share1.0.18 + +Pull Requests that document the changes (include PR ids): +(https://github.com/ESCOMP/ctsm/pull) + + #2396 Explain residue removal + #2385 Update externals + #2394 Fix tillage instructions and images + #2389 Minor docs updates + +=============================================================== +=============================================================== Tag name: ctsm5.1.dev171 Originator(s): olyson (Keith Oleson,UCAR/TSS) Date: Mon 04 Mar 2024 10:33:55 AM MST diff --git a/doc/ChangeSum b/doc/ChangeSum index 32b60a502f..c9cf2183f4 100644 --- a/doc/ChangeSum +++ b/doc/ChangeSum @@ -1,5 +1,6 @@ Tag Who Date Summary ============================================================================================================================ + ctsm5.1.dev172 erik 03/12/2024 Merge b4b-dev ctsm5.1.dev171 slevis 03/01/2024 Set initial t_soisno=272 for soils and 274K for urban road ctsm5.1.dev170 samrabin 02/28/2024 Add hillslope hydrology ctsm5.1.dev169 samrabin 02/22/2024 Merge b4b-dev