Merge pull request ESCOMP#2628 from ekluzek/merge-tmpdev-20240627

Bring temporary branch to main-development

.git-blame-ignore-revs

@@ -46,3 +46,4 @@ aa04d1f7d86cc2503b98b7e2b2d84dbfff6c316b
 6c6f57e948bfa31e60b383536cc21663fedb8b70
 9660667b1267dcd4150889f5f39db540158be74a
 665cf86102e09b4c4c5a140700676dca23bc55a9
+045d90f1d80f713eb3ae0ac58f6c2352937f1eb0

.gitignore

@@ -1,7 +1,3 @@
-# ignore svn directories
-**/.svn/**
-.svn/
-
 # netcdf files
 *.nc
 # but don't ignore netcdf files here:

.gitmodules

@@ -1,90 +1,130 @@
+# This is a git submodule file with additional support for
+# git-fleximod (https://github.com/ESMCI/git-fleximod)
+#
+# The additional flags supported by git-fleximod are
+# fxtag - the tag associated with the submodule, this tag can be tested for
+#         consistancy with the submodule hash using git-fleximod status
+#         the hash can be updated to the tag using git-fleximod update
+#
+# fxrequired - indicates if a given submodule should be checked out on install
+#              submoudules can be toplevel or internal and required or optional
+#              toplevel means that the submodule should only be checked out if the
+#              module is the toplevel of the git repo (is not a submodule itself)
+#              internal means that the submodule is needed by the component whether
+#              the component is toplevel or the submodule of another repo
+#              required means that the submodule should always be checked out
+#              optional means that the submodule should only be checked out if the
+#              optional flag is provided to git-fleximod or the submodule name is
+#              explicitly listed on the git-fleximod command line.
+#
+# fxsparse -   this is a path to a git sparse checkout file indicating that the
+#              submodule should be checked out in sparse mode
+#
+# fxDONOTUSEurl    -   this field is used by git-fleximod test to insure that the url is pointing
+#              to the official url of the repo and not to an unofficial fork.
+#              It is intended for use of github workflows to test commits to protected
+#              repository branches.
+#
 [submodule "fates"]
 path = src/fates
 url = https://github.com/NGEET/fates
 fxtag = sci.1.73.0_api.35.0.0
 fxrequired = AlwaysRequired
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/NCAR/fates-release
 
 [submodule "cism"]
 path = components/cism
 url = https://github.com/ESCOMP/CISM-wrapper
 fxtag = cismwrap_2_2_001
 fxrequired = ToplevelRequired
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/CISM-wrapper
 
 [submodule "rtm"]
 path = components/rtm
 url = https://github.com/ESCOMP/RTM
 fxtag = rtm1_0_79
 fxrequired = ToplevelRequired
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/RTM
 
 [submodule "mosart"]
 path = components/mosart
 url = https://github.com/ESCOMP/MOSART
 fxtag = mosart1_0_49
 fxrequired = ToplevelRequired
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/MOSART
 
 [submodule "mizuRoute"]
 path = components/mizuRoute
 url = https://github.com/ESCOMP/mizuRoute
 fxtag = cesm-coupling.n02_v2.1.2
 fxrequired = ToplevelRequired
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute
 
 [submodule "ccs_config"]
 path = ccs_config
 url = https://github.com/ESMCI/ccs_config_cesm.git
 fxtag = ccs_config_cesm0.0.106
 fxrequired = ToplevelRequired
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git
 
 [submodule "cime"]
 path = cime
 url = https://github.com/ESMCI/cime
 fxtag = cime6.0.246
 fxrequired = ToplevelRequired
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESMCI/cime
 
 [submodule "cmeps"]
 path = components/cmeps
 url = https://github.com/ESCOMP/CMEPS.git
 fxtag = cmeps0.14.63
 fxrequired = ToplevelRequired
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git
 
 [submodule "cdeps"]
 path = components/cdeps
 url = https://github.com/ESCOMP/CDEPS.git
 fxtag = cdeps1.0.34
 fxrequired = ToplevelRequired
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS.git
 
 [submodule "share"]
 path = share
 url = https://github.com/ESCOMP/CESM_share
 fxtag = share1.0.19
 fxrequired = ToplevelRequired
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESCOMP/CESM_share
 
 [submodule "mct"]
 path = libraries/mct
 url = https://github.com/MCSclimate/MCT
 fxtag = MCT_2.11.0
 fxrequired = ToplevelRequired
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/MCSclimate/MCT
 
 [submodule "parallelio"]
 path = libraries/parallelio
 url = https://github.com/NCAR/ParallelIO
 fxtag = pio2_6_2
 fxrequired = ToplevelRequired
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/NCAR/ParallelIO
 
 [submodule "doc-builder"]
 path = doc/doc-builder
 url = https://github.com/ESMCI/doc-builder
 fxtag = v1.0.8
 fxrequired = ToplevelOptional
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESMCI/doc-builder

.lib/git-fleximod/git_fleximod/cli.py

@@ -2,7 +2,7 @@
 import argparse
 from git_fleximod import utils
 
-__version__ = "0.7.5"
+__version__ = "0.7.8"
 
 def find_root_dir(filename=".gitmodules"):
     """ finds the highest directory in tree
@@ -21,12 +21,12 @@ def find_root_dir(filename=".gitmodules"):
         attempt = dl / filename
         if attempt.is_file():
             return str(dl)
-    utils.fatal_error("No .gitmodules found in directory tree")
+    return None
 
 
 def get_parser():
     description = """
-    %(prog)s manages checking out groups of gitsubmodules with addtional support for Earth System Models
+    %(prog)s manages checking out groups of gitsubmodules with additional support for Earth System Models
     """
     parser = argparse.ArgumentParser(
         description=description, formatter_class=argparse.RawDescriptionHelpFormatter

.lib/git-fleximod/git_fleximod/git_fleximod.py

@@ -263,6 +263,21 @@ def single_submodule_checkout(
 
     return
 
+def add_remote(git, url):
+    remotes = git.git_operation("remote", "-v")
+    newremote = "newremote.00"
+    if url in remotes:
+        for line in remotes:
+            if url in line and "fetch" in line:
+                newremote = line.split()[0]
+                break
+    else:
+        i = 0
+        while "newremote" in remotes:
+            i = i + 1
+            newremote = f"newremote.{i:02d}"
+        git.git_operation("remote", "add", newremote, url)
+    return newremote
 
 def submodules_status(gitmodules, root_dir, toplevel=False):
     testfails = 0
@@ -271,6 +286,7 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
     for name in gitmodules.sections():
         path = gitmodules.get(name, "path")
         tag = gitmodules.get(name, "fxtag")
+        url = gitmodules.get(name, "url")
         required = gitmodules.get(name, "fxrequired")
         level = required and "Toplevel" in required
         if not path:
@@ -280,43 +296,54 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
         if not os.path.exists(os.path.join(newpath, ".git")):
             rootgit = GitInterface(root_dir, logger)
             # submodule commands use path, not name
-            url = gitmodules.get(name, "url")
             url = url.replace("[email protected]:", "https://github.com/")
             tags = rootgit.git_operation("ls-remote", "--tags", url)
+            result = rootgit.git_operation("submodule","status",newpath).split()
+            ahash = None
+            if result:
+                ahash = result[0][1:]
+            hhash = None
             atag = None
+
             needsupdate += 1
             if not toplevel and level:
                 continue
             for htag in tags.split("\n"):
-                if tag and tag in htag:
+                if htag.endswith('^{}'):
+                    htag = htag[:-3]
+                if ahash and not atag and ahash in htag:
                     atag = (htag.split()[1])[10:]
+                if tag and not hhash and htag.endswith(tag):
+                    hhash = htag.split()[0]
+                if hhash and atag:
                     break
-            if tag and tag == atag:
-                print(f"e {name:>20} not checked out, aligned at tag {tag}")
+            optional = " (optional)" if required and "Optional" in required else ""
+            if tag and (ahash == hhash or atag == tag):
+                print(f"e {name:>20} not checked out, aligned at tag {tag}{optional}")
             elif tag:
                 ahash = rootgit.git_operation(
                     "submodule", "status", "{}".format(path)
                 ).rstrip()
                 ahash = ahash[1 : len(tag) + 1]
                 if tag == ahash:
-                    print(f"e {name:>20} not checked out, aligned at hash {ahash}")
+                    print(f"e {name:>20} not checked out, aligned at hash {ahash}{optional}")
                 else:
                     print(
-                        f"e {name:>20} not checked out, out of sync at tag {atag}, expected tag is {tag}"
+                        f"e {name:>20} not checked out, out of sync at tag {atag}, expected tag is {tag}{optional}"
                     )
                     testfails += 1
             else:
-                print(f"e {name:>20} has no fxtag defined in .gitmodules")
+                print(f"e {name:>20} has no fxtag defined in .gitmodules{optional}")
                 testfails += 1
         else:
             with utils.pushd(newpath):
                 git = GitInterface(newpath, logger)
                 atag = git.git_operation("describe", "--tags", "--always").rstrip()
-                part =  git.git_operation("status").partition("\n")[0]
-                # fake hash to initialize
-                ahash = "xxxx"
-                if part:
-                    ahash = part.split()[-1]
+                ahash =  git.git_operation("rev-list", "HEAD").partition("\n")[0]
+                rurl = git.git_operation("ls-remote","--get-url").rstrip()
+                if rurl != url:
+                    remote = add_remote(git, url)
+                    git.git_operation("fetch", remote)
                 if tag and atag == tag:
                     print(f"  {name:>20} at tag {tag}")
                 elif tag and ahash[: len(tag)] == tag:
@@ -335,7 +362,7 @@ def submodules_status(gitmodules, root_dir, toplevel=False):
                     )
                     testfails += 1
 
-                status = git.git_operation("status", "--ignore-submodules")
+                status = git.git_operation("status", "--ignore-submodules", "-uno")
                 if "nothing to commit" not in status:
                     localmods = localmods + 1
                     print("M" + textwrap.indent(status, "                      "))
@@ -357,11 +384,11 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
         path = gitmodules.get(name, "path")
         url = gitmodules.get(name, "url")
         logger.info(
-            "name={} path={} url={} fxtag={} requiredlist={}".format(
+            "name={} path={} url={} fxtag={} requiredlist={} ".format(
                 name, os.path.join(root_dir, path), url, fxtag, requiredlist
             )
         )
-        #        if not os.path.exists(os.path.join(root_dir,path, ".git")):
+
         fxrequired = gitmodules.get(name, "fxrequired")
         assert fxrequired in fxrequired_allowed_values()
         rgit = GitInterface(root_dir, logger)
@@ -409,19 +436,7 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                 upstream = git.git_operation("ls-remote", "--get-url").rstrip()
                 newremote = "origin"
                 if upstream != url:
-                    # TODO - this needs to be a unique name
-                    remotes = git.git_operation("remote", "-v")
-                    if url in remotes:
-                        for line in remotes:
-                            if url in line and "fetch" in line:
-                                newremote = line.split()[0]
-                                break
-                    else:
-                        i = 0
-                        while newremote in remotes:
-                            i = i + 1
-                            newremote = f"newremote.{i:02d}"
-                        git.git_operation("remote", "add", newremote, url)
+                    add_remote(git, url)
 
                 tags = git.git_operation("tag", "-l")
                 if fxtag and fxtag not in tags:
@@ -439,6 +454,8 @@ def submodules_update(gitmodules, root_dir, requiredlist, force):
                     print(f"{name:>20} up to date.")
 
 
+
+
 def local_mods_output():
     text = """\
     The submodules labeled with 'M' above are not in a clean state.
@@ -560,10 +577,12 @@ def main():
 
     logger.info("action is {} root_dir={} file_name={}".format(action, root_dir, file_name))
 
-    if not os.path.isfile(os.path.join(root_dir, file_name)):
-        file_path = utils.find_upwards(root_dir, file_name)
+    if not root_dir or not os.path.isfile(os.path.join(root_dir, file_name)):
+        if root_dir:
+            file_path = utils.find_upwards(root_dir, file_name)
 
-        if file_path is None:
+        if root_dir is None or file_path is None:
+            root_dir = "."
             utils.fatal_error(
                 "No {} found in {} or any of it's parents".format(file_name, root_dir)
             )

.lib/git-fleximod/git_fleximod/gitmodules.py

@@ -59,7 +59,7 @@ def get(self, name, option, raw=False, vars=None, fallback=None):
         Uses the parent class's get method to access the value.
         Handles potential errors if the section or option doesn't exist.
         """
-        self.logger.debug("get called {} {}".format(name, option))
+        self.logger.debug("git get called {} {}".format(name, option))
         section = f'submodule "{name}"'
         try:
             return ConfigParser.get(

.lib/git-fleximod/git_fleximod/utils.py

@@ -241,12 +241,12 @@ def _hanging_msg(working_directory, command):
 has taken {hanging_sec} seconds. It may be hanging.
 
 The command will continue to run, but you may want to abort
-git-fleximod with ^C and investigate. A possible cause of hangs is
-when svn or git require authentication to access a private
-repository. On some systems, svn and git requests for authentication
-information will not be displayed to the user. In this case, the program
-will appear to hang. Ensure you can run svn and git manually and access
-all repositories without entering your authentication information.
+git-fleximod with ^C and investigate. A possible cause of hangs is git
+requires authentication to access a private repository. On some
+systems, git requests for authentication information will not
+be displayed to the user. In this case, the program will appear to
+hang. Ensure you can run git manually and access all
+repositories without entering your authentication information.
 
 """.format(
             command=command,

.lib/git-fleximod/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "git-fleximod"
-version = "0.7.5"
+version = "0.7.8"
 description = "Extended support for git-submodule and git-sparse-checkout"
 authors = ["Jim Edwards <[email protected]>"]
 maintainers = ["Jim Edwards <[email protected]>"]

.lib/git-fleximod/tbump.toml

@@ -2,7 +2,7 @@
 github_url = "https://github.com/jedwards4b/git-fleximod/"
 
 [version]
-current = "0.7.5"
+current = "0.7.8"
 
 # Example of a semver regexp.
 # Make sure this matches current_version before

.lib/git-fleximod/tests/conftest.py

@@ -32,7 +32,7 @@ def logger():
      "submodule_name": "test_optional",
      "status1" : "test_optional MPIserial_2.5.0-3-gd82ce7c is out of sync with .gitmodules MPIserial_2.4.0",
      "status2" : "test_optional at tag MPIserial_2.4.0",
-     "status3" : "test_optional not checked out, aligned at tag MPIserial_2.4.0",
+     "status3" : "test_optional not checked out, out of sync at tag None, expected tag is MPIserial_2.4.0 (optional)",
      "status4" : "test_optional at tag MPIserial_2.4.0",
      "gitmodules_content": """
      [submodule "test_optional"]
@@ -46,7 +46,7 @@ def logger():
      "submodule_name": "test_alwaysoptional",
      "status1" : "test_alwaysoptional MPIserial_2.3.0 is out of sync with .gitmodules e5cf35c",
      "status2" : "test_alwaysoptional at hash e5cf35c",
-     "status3" : "test_alwaysoptional not checked out, out of sync at tag MPIserial_2.3.0",
+     "status3" : "out of sync at tag None, expected tag is e5cf35c",
      "status4" : "test_alwaysoptional at hash e5cf35c",
      "gitmodules_content": """
     [submodule "test_alwaysoptional"]