Merge pull request #280 from ESCOMP/cesm3.0-alphabranch

cesm3_0_alpha3b

.gitmodules

@@ -30,22 +30,22 @@
         path = ccs_config
         url = https://github.com/ESMCI/ccs_config_cesm.git
         fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git
-        fxtag = ccs_config_cesm1.0.1
+        fxtag = ccs_config_cesm1.0.6
         fxrequired = ToplevelRequired
 
 [submodule "cime"]
         path = cime
         url = https://github.com/ESMCI/cime
         fxDONOTUSEurl = https://github.com/ESMCI/cime
-        fxtag = cime6.1.0
+        fxtag = cime6.1.22
         fxrequired = ToplevelRequired
 
 [submodule "fms"]
         path = libraries/FMS
         url = https://github.com/ESCOMP/FMS_interface
         fxDONOTUSEurl = https://github.com/ESCOMP/FMS_interface
         fxrequired = ToplevelRequired
-        fxtag = fi_240516
+        fxtag = fi_240828
 
 [submodule "share"]
         path = share
@@ -66,7 +66,7 @@
         url = https://github.com/ESCOMP/CTSM
         fxDONOTUSEurl = https://github.com/ESCOMP/CTSM
         fxrequired = ToplevelRequired
-        fxtag = ctsm5.2.009
+        fxtag = ctsm5.2.027
 
 [submodule "cice"]
         path = components/cice
@@ -80,7 +80,7 @@
         url = https://github.com/ESCOMP/MOM_interface
         fxDONOTUSEurl = https://github.com/ESCOMP/MOM_interface
         fxrequired = ToplevelRequired
-        fxtag = mi_240705
+        fxtag = mi_240822
 
 [submodule "cism"]
         path = components/cism
@@ -94,14 +94,14 @@
         url = https://github.com/ESCOMP/CDEPS
         fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS
         fxrequired = ToplevelRequired
-        fxtag = cdeps1.0.43
+        fxtag = cdeps1.0.49
 
 [submodule "cmeps"]
         path = components/cmeps
         url = https://github.com/ESCOMP/CMEPS.git
         fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git
         fxrequired = ToplevelRequired
-        fxtag = cmeps1.0.6
+        fxtag = cmeps1.0.16
 
 [submodule "rtm"]
         path = components/rtm
@@ -114,7 +114,7 @@
         path = components/ww3
         url = https://github.com/ESCOMP/WW3_interface
         fxDONOTUSEurl = https://github.com/ESCOMP/WW3_interface
-        fxtag = ww3i_0.0.2
+        fxtag = ww3i_0.0.3
         fxrequired = ToplevelRequired
 
 [submodule "mizuroute"]
@@ -136,7 +136,7 @@
         url = https://github.com/NCAR/ParallelIO
         fxDONOTUSEurl = https://github.com/NCAR/ParallelIO
         fxrequired = ToplevelRequired
-        fxtag = pio2_6_2
+        fxtag = pio2_6_3
 
 [submodule "mpi-serial"]
         path = libraries/mpi-serial

.lib/git-fleximod/git_fleximod/cli.py

@@ -2,7 +2,7 @@
 import argparse
 from git_fleximod import utils
 
-__version__ = "0.8.4"
+__version__ = "0.9.0"
 
 def find_root_dir(filename=".gitmodules"):
     """ finds the highest directory in tree

.lib/git-fleximod/git_fleximod/git_fleximod.py

@@ -124,8 +124,8 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master
     # set the repository remote
 
     logger.info("Setting remote origin in {}/{}".format(root_dir, path))
-    status = sprepo_git.git_operation("remote", "-v")
-    if url not in status:
+    _, remotelist = sprepo_git.git_operation("remote", "-v")
+    if url not in remotelist:
         sprepo_git.git_operation("remote", "add", "origin", url)
 
     topgit = os.path.join(gitroot, ".git")
@@ -213,7 +213,7 @@ def submodules_status(gitmodules, root_dir, toplevel=False, depth=0):
 
 def git_toplevelroot(root_dir, logger):
     rgit = GitInterface(root_dir, logger)
-    superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
+    _, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
     return superroot
 
 def submodules_update(gitmodules, root_dir, requiredlist, force):

.lib/git-fleximod/git_fleximod/gitinterface.py

@@ -59,11 +59,12 @@ def git_operation(self, operation, *args, **kwargs):
         command = self._git_command(operation, *newargs)
         if isinstance(command, list):
             try:
-                return utils.execute_subprocess(command, output_to_caller=True)
+                status, output = utils.execute_subprocess(command, status_to_caller=True, output_to_caller=True)
+                return status, output.rstrip()
             except Exception as e:
                 sys.exit(e)
         else:
-            return command
+            return 0, command
 
     def config_get_value(self, section, name):
         if self._use_module:

.lib/git-fleximod/git_fleximod/submodule.py

@@ -60,8 +60,9 @@ def status(self):
         if not os.path.exists(os.path.join(smpath, ".git")):
             rootgit = GitInterface(self.root_dir, self.logger)
             # submodule commands use path, not name
-            tags = rootgit.git_operation("ls-remote", "--tags", self.url)
-            result = rootgit.git_operation("submodule","status",smpath).split()
+            status, tags = rootgit.git_operation("ls-remote", "--tags", self.url)
+            status, result = rootgit.git_operation("submodule","status",smpath)
+            result = result.split()
 
             if result:
                 ahash = result[0][1:]
@@ -80,9 +81,9 @@ def status(self):
                 result = f"e {self.name:>20} not checked out, aligned at tag {self.fxtag}{optional}"
                 needsupdate = True
             elif self.fxtag:
-                ahash = rootgit.git_operation(
+                status, ahash = rootgit.git_operation(
                     "submodule", "status", "{}".format(self.path)
-                ).rstrip()
+                )
                 ahash = ahash[1 : len(self.fxtag) + 1]
                 if self.fxtag == ahash:
                     result = f"e {self.name:>20} not checked out, aligned at hash {ahash}{optional}"
@@ -96,14 +97,15 @@ def status(self):
         else:
             with utils.pushd(smpath):
                 git = GitInterface(smpath, self.logger)
-                remote = git.git_operation("remote").rstrip()
+                status, remote = git.git_operation("remote")
                 if remote == '':
                     result = f"e {self.name:>20} has no associated remote"
                     testfails = True
                     needsupdate = True
                     return result, needsupdate, localmods, testfails                    
-                rurl = git.git_operation("ls-remote","--get-url").rstrip()
-                line = git.git_operation("log", "--pretty=format:\"%h %d\"").partition('\n')[0]
+                status, rurl = git.git_operation("ls-remote","--get-url")
+                status, lines = git.git_operation("log", "--pretty=format:\"%h %d\"")
+                line = lines.partition('\n')[0]
                 parts = line.split()
                 ahash = parts[0][1:]
                 atag = None
@@ -120,7 +122,7 @@ def status(self):
 
 
                 #print(f"line is {line} ahash is {ahash} atag is {atag} {parts}")
-                #                atag = git.git_operation("describe", "--tags", "--always").rstrip()
+                #                atag = git.git_operation("describe", "--tags", "--always")
                 # ahash =  git.git_operation("rev-list", "HEAD").partition("\n")[0]
 
                 recurse = False
@@ -149,10 +151,10 @@ def status(self):
                         result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {ahash}"
                     testfails = False
 
-                status = git.git_operation("status", "--ignore-submodules", "-uno")
-                if "nothing to commit" not in status:
+                status, output = git.git_operation("status", "--ignore-submodules", "-uno")
+                if "nothing to commit" not in output:
                     localmods = True
-                    result = "M" + textwrap.indent(status, "                      ")
+                    result = "M" + textwrap.indent(output, "                      ")
 #        print(f"result {result} needsupdate {needsupdate} localmods {localmods} testfails {testfails}")
         return result, needsupdate, localmods, testfails
 
@@ -171,10 +173,11 @@ def _add_remote(self, git):
         Returns:
             str: The name of the new remote if added, or the name of the existing remote that matches the submodule's URL.
         """ 
-        remotes = git.git_operation("remote", "-v").splitlines()
+        status, remotes = git.git_operation("remote", "-v")
+        remotes = remotes.splitlines()
         upstream = None
         if remotes:
-            upstream = git.git_operation("ls-remote", "--get-url").rstrip()
+            status, upstream = git.git_operation("ls-remote", "--get-url")
             newremote = "newremote.00"
             tmpurl = self.url.replace("[email protected]:", "https://github.com/")
             line = next((s for s in remotes if self.url in s or tmpurl in s), None)
@@ -214,12 +217,19 @@ def sparse_checkout(self):
         """ 
         self.logger.info("Called sparse_checkout for {}".format(self.name))
         rgit = GitInterface(self.root_dir, self.logger)
-        superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
+        status, superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree")
         if superroot:
             gitroot = superroot.strip()
         else:
             gitroot = self.root_dir.strip()
-        assert os.path.isdir(os.path.join(gitroot, ".git"))
+        # Now need to move the .git dir to the submodule location
+        rootdotgit = os.path.join(self.root_dir, ".git")
+        while os.path.isfile(rootdotgit):
+            with open(rootdotgit) as f:
+                line = f.readline().rstrip()
+                if line.startswith("gitdir: "):
+                    rootdotgit = os.path.abspath(os.path.join(self.root_dir,line[8:]))
+        assert os.path.isdir(rootdotgit)
         # first create the module directory
         if not os.path.isdir(os.path.join(self.root_dir, self.path)):
             os.makedirs(os.path.join(self.root_dir, self.path))
@@ -244,8 +254,8 @@ def sparse_checkout(self):
         # set the repository remote
 
         self.logger.info("Setting remote origin in {}/{}".format(self.root_dir, self.path))
-        status = sprepo_git.git_operation("remote", "-v")
-        if self.url not in status:
+        status, remotes = sprepo_git.git_operation("remote", "-v")
+        if self.url not in remotes:
             sprepo_git.git_operation("remote", "add", "origin", self.url)
 
         topgit = os.path.join(gitroot, ".git")
@@ -256,43 +266,43 @@ def sparse_checkout(self):
                     os.path.join(self.root_dir, f.read().split()[1]),
                     start=os.path.join(self.root_dir, self.path),
                 )
-                topgit = os.path.join(gitpath, "modules")
+                rootdotgit = os.path.join(gitpath, "modules", self.name)
         else:
-            topgit = os.path.relpath(
-                os.path.join(self.root_dir, ".git", "modules"),
+            rootdotgit = os.path.relpath(
+                os.path.join(self.root_dir, ".git", "modules", self.name),
                 start=os.path.join(self.root_dir, self.path),
             )
 
-        with utils.pushd(sprep_repo):
-            if not os.path.isdir(topgit):
-                os.makedirs(topgit)
-        topgit += os.sep + self.name
-
         if os.path.isdir(os.path.join(self.root_dir, self.path, ".git")):
             with utils.pushd(sprep_repo):
-                if os.path.isdir(os.path.join(topgit,".git")):
-                    shutil.rmtree(os.path.join(topgit,".git"))
-                shutil.move(".git", topgit)
+                if os.path.isdir(os.path.join(rootdotgit,".git")):
+                    shutil.rmtree(os.path.join(rootdotgit,".git"))
+                shutil.move(".git", rootdotgit)
                 with open(".git", "w") as f:
-                    f.write("gitdir: " + os.path.relpath(topgit))
-                #    assert(os.path.isdir(os.path.relpath(topgit, start=sprep_repo)))
-                gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout"))
+                    f.write("gitdir: " + os.path.relpath(rootdotgit))
+                infodir = os.path.join(rootdotgit, "info")
+                if not os.path.isdir(infodir):
+                    os.makedirs(infodir)
+                gitsparse = os.path.abspath(os.path.join(infodir, "sparse-checkout"))
             if os.path.isfile(gitsparse):
                 self.logger.warning(
-                    "submodule {} is already initialized {}".format(self.name, topgit)
+                    "submodule {} is already initialized {}".format(self.name, rootdotgit)
                 )
                 return
 
             with utils.pushd(sprep_repo):
                 if os.path.isfile(self.fxsparse):
+
                     shutil.copy(self.fxsparse, gitsparse)
 
 
         # Finally checkout the repo
         sprepo_git.git_operation("fetch", "origin", "--tags")
-        sprepo_git.git_operation("checkout", self.fxtag)
-
-        print(f"Successfully checked out {self.name:>20} at {self.fxtag}")
+        status,_ = sprepo_git.git_operation("checkout", self.fxtag)
+        if status:
+            print(f"Error checking out {self.name:>20} at {self.fxtag}")
+        else:
+            print(f"Successfully checked out {self.name:>20} at {self.fxtag}")
         rgit.config_set_value(f'submodule "{self.name}"', "active", "true")
         rgit.config_set_value(f'submodule "{self.name}"', "url", self.url)
         rgit.config_set_value(f'submodule "{self.name}"', "path", self.path)
@@ -342,15 +352,15 @@ def update(self):
                     git.git_operation("clone", self.url, self.path)
                     smgit = GitInterface(repodir, self.logger)
                     if not tag:
-                        tag = smgit.git_operation("describe", "--tags", "--always").rstrip()
+                        status, tag = smgit.git_operation("describe", "--tags", "--always")
                     smgit.git_operation("checkout", tag)
                     # Now need to move the .git dir to the submodule location
                     rootdotgit = os.path.join(self.root_dir, ".git")
                     if os.path.isfile(rootdotgit):
                         with open(rootdotgit) as f:
                             line = f.readline()
                             if line.startswith("gitdir: "):
-                                rootdotgit = line[8:].rstrip()
+                                rootdotgit = line[8:]
 
                     newpath = os.path.abspath(os.path.join(self.root_dir, rootdotgit, "modules", self.name))
                     if os.path.exists(newpath):
@@ -393,15 +403,16 @@ def update(self):
                 git = GitInterface(submoddir, self.logger)
                 # first make sure the url is correct
                 newremote = self._add_remote(git)
-                tags = git.git_operation("tag", "-l")
+                status, tags = git.git_operation("tag", "-l")
                 fxtag = self.fxtag
                 if fxtag and fxtag not in tags:
                     git.git_operation("fetch", newremote, "--tags")
-                atag = git.git_operation("describe", "--tags", "--always").rstrip()
+                status, atag = git.git_operation("describe", "--tags", "--always")
                 if fxtag and fxtag != atag:
                     try:
-                        git.git_operation("checkout", fxtag)
-                        print(f"{self.name:>20} updated to {fxtag}")
+                        status, _ = git.git_operation("checkout", fxtag)
+                        if not status:
+                            print(f"{self.name:>20} updated to {fxtag}")
                     except Exception as error:
                         print(error)
 

.lib/git-fleximod/git_fleximod/utils.py

@@ -307,12 +307,15 @@ def execute_subprocess(commands, status_to_caller=False, output_to_caller=False)
         # simple status check. If returning, it is the callers
         # responsibility determine if an error occurred and handle it
         # appropriately.
-        if not return_to_caller:
-            msg_context = (
-                "Process did not run successfully; "
-                "returned status {0}".format(error.returncode)
-            )
-            msg = failed_command_msg(msg_context, commands, output=error.output)
+        msg_context = (
+            "Process did not run successfully; "
+            "returned status {0}".format(error.returncode)
+        )
+        msg = failed_command_msg(msg_context, commands, output=error.output)
+        if return_to_caller:
+            logging.warning(error)
+            logging.warning(msg)
+        else:
             logging.error(error)
             logging.error(msg)
             log_process_output(error.output)

.lib/git-fleximod/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "git-fleximod"
-version = "0.8.4"
+version = "0.9.0"
 description = "Extended support for git-submodule and git-sparse-checkout"
 authors = ["Jim Edwards <[email protected]>"]
 maintainers = ["Jim Edwards <[email protected]>"]

.lib/git-fleximod/tbump.toml

@@ -2,7 +2,7 @@
 github_url = "https://github.com/jedwards4b/git-fleximod/"
 
 [version]
-current = "0.8.4"
+current = "0.9.0"
 
 # Example of a semver regexp.
 # Make sure this matches current_version before