#221 Merge branch 'main' of github.com:NOAA-GFDL/fre-cli into 221.fo…

…rce-checkout-and-compile

fre/make/create_docker_script.py

@@ -9,7 +9,7 @@
 from .gfdlfremake import varsfre, targetfre, yamlfre, buildDocker
 import fre.yamltools.combine_yamls as cy
 
-def dockerfile_write_steps(yaml_obj,img,run_env,target,mkTemplate,td,cr,cb,cd):
+def dockerfile_write_steps(yaml_obj,img,run_env,target,mkTemplate,s2i,td,cr,cb,cd):
     """
     Go through steps to write the Dockerfile
     """
@@ -18,7 +18,8 @@ def dockerfile_write_steps(yaml_obj,img,run_env,target,mkTemplate,td,cr,cb,cd):
                                         libs = yaml_obj["container_addlibs"],
                                         RUNenv = run_env,
                                         target = target,
-                                        mkTemplate = mkTemplate)
+                                        mkTemplate = mkTemplate
+                                        stage2base = s2i)
 
     dockerBuild.writeDockerfileCheckout("checkout.sh", td+"/checkout.sh")
     dockerBuild.writeDockerfileMakefile(td+"/Makefile", td+"/linkline.sh")
@@ -77,6 +78,7 @@ def dockerfile_create(yamlfile,platform,target,execute,force_dockerfile):
             ## Check for type of build
             if platform["container"] is True:
                 image=modelYaml.platforms.getContainerImage(platformName)
+                stage2image = modelYaml.platforms.getContainer2base(platformName)
                 bld_dir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/exec"
                 tmp_dir = "tmp/"+platformName
                 curr_dir = os.getcwd()
@@ -86,6 +88,7 @@ def dockerfile_create(yamlfile,platform,target,execute,force_dockerfile):
                                            run_env = platform["RUNenv"],
                                            target = targetObject,
                                            mkTemplate = platform["mkTemplate"],
+                                           s2i = stage2image,
                                            td = tmp_dir,
                                            cr = platform["containerRun"],
                                            cb = platform["containerBuild"],

fre/make/gfdlfremake/buildDocker.py

@@ -18,8 +18,12 @@ class container():
         - RUNenv : The commands that have to be run at
                    the beginning of a RUN in the dockerfile
                    to set up the environment
+        - target : The FRE target
+        - mkTemplate: The mkmf template to use
+        - stage2base: The base for the second stage. Empty
+                      string if there is no second stage
     """
-    def __init__(self,base,exp,libs,RUNenv,target,mkTemplate):
+    def __init__(self,base,exp,libs,RUNenv,target,mkTemplate,stage2base):
         """
         Initialize variables and write to the dockerfile
         """
@@ -31,6 +35,7 @@ def __init__(self,base,exp,libs,RUNenv,target,mkTemplate):
         self.mkmf = True
         self.target = target
         self.template = mkTemplate
+        self.stage2base = stage2base
 
         # Set up spack loads in RUN commands in dockerfile
         if RUNenv == "":
@@ -54,15 +59,15 @@ def __init__(self,base,exp,libs,RUNenv,target,mkTemplate):
                        " && src_dir="+self.src+" \\ \n",
                        " && mkmf_template="+self.template+ " \\ \n"]
         self.d=open("Dockerfile","w")
-        self.d.writelines("FROM "+self.base+" \n")
-        if self.base == "ecpe4s/noaa-intel-prototype:2023.09.25":
-            self.prebuild = '''RUN 
-            '''
-            self.postbuild = '''
-            '''
-            self.secondstage = '''
-            '''
-
+        self.d.writelines("FROM "+self.base+" as builder\n")
+        ## Set up the second stage build list of lines to add
+        if self.stage2base == "":
+            self.secondstage = ["\n"]
+        else:
+            self.secondstage = [f"FROM {self.stage2base} as final\n",
+                                f"COPY --from=builder  {self.src} {self.src}\n",
+                                f"COPY --from=builder {self.bld} {self.bld}\n",
+                                f"ENV PATH=$PATH:{self.bld}\n"]
     def writeDockerfileCheckout(self, cScriptName, cOnDisk):
         """
         Brief: writes to the checkout part of the Dockerfile and sets up the compile
@@ -201,6 +206,9 @@ def writeRunscript(self,RUNenv,containerRun,runOnDisk):
             self.d.write(" cd "+self.bld+" && make -j 4 "+self.target.getmakeline_add()+"\n")
         else:
             self.d.write(" && cd "+self.bld+" && make -j 4 "+self.target.getmakeline_add()+"\n")
+        ## Write any second stage lines here 
+        for l in self.secondstage:
+            self.d.write(l)
         self.d.write('ENTRYPOINT ["/bin/bash"]')
         self.d.close()
 

fre/make/gfdlfremake/platformfre.py

@@ -41,35 +41,49 @@ def __init__(self,platforminfo):
             ## Check if we are working with a container and get the info for that
             try:
                 p["container"]
+            ## When not doing a container build, this should all be set to empty strings and Falses
             except:
                 p["container"] = False
                 p["RUNenv"] = [""]
                 p["containerBuild"] = ""
                 p["containerRun"] = ""
-                p["containerViews"] = False
                 p["containerBase"] = ""
-                p["container2step"] = ""
+                p["container2step"] = False
+                p["container2base"] = ""
             if p["container"]:
                 ## Check the container builder
                 try:
                     p["containerBuild"]
                 except:
-                    raise Exception("You must specify the program used to build the container (containerBuild) on the "+p["name"]+" platform in the file "+fname+"\n")
+                    raise Exception("Platform "+p["name"]+": You must specify the program used to build the container (containerBuild) on the "+p["name"]+" platform in the file "+fname+"\n")
                 if p["containerBuild"] != "podman" and p["containerBuild"] != "docker":
-                    raise ValueError("Container builds only supported with docker or podman, but you listed "+p["containerBuild"]+"\n")
-## Check for container environment set up for RUN commands
+                    raise ValueError("Platform "+p["name"]+": Container builds only supported with docker or podman, but you listed "+p["containerBuild"]+"\n")
+                ## Get the name of the base container
                 try:
                     p["containerBase"]
-                except NameError:
-                    print("You must specify the base container you wish to use to build your application")
-                try:
-                    p["containerViews"]
                 except:
-                    p["containerViews"] = False
+                    raise NameError("Platform "+p["name"]+": You must specify the base container you wish to use to build your application")
+                ## Check if this is a 2 step (multi stage) build
                 try:
                     p["container2step"]
                 except:
-                    p["container2step"] = ""
+                    p["container2step"] = False
+                ## Get the base for the second stage of the build
+                if p["container2step"]:
+                    try:
+                        p["container2base"]
+                    except:
+                        raise NameError ("Platform "+p["name"]+": container2step is True, so you must define a container2base\n")
+                    ## Check if there is anything special to copy over
+                else:
+                    ## There should not be a second base if this is not a 2 step build
+                    try:
+                        p["container2base"]
+                    except:
+                        p["container2base"] = ""
+                    else:
+                        raise ValueError ("Platform "+p["name"]+": You defined container2base "+p["container2base"]+" but container2step is False\n")
+                ## Get any commands to execute in the dockerfile RUN command
                 try:
                     p["RUNenv"]
                 except:
@@ -82,6 +96,7 @@ def __init__(self,platforminfo):
                 if p["containerRun"] != "apptainer" and p["containerRun"] != "singularity":
                     raise ValueError("Container builds only supported with apptainer, but you listed "+p["containerRun"]+"\n")
             else:
+                ## Find the location of the mkmf template
                 try:
                     p["mkTemplate"]
                 except:
@@ -119,7 +134,6 @@ def getContainerInfoFromName(self,name):
                 p["RUNenv"], \
                 p["containerBuild"], \
                 p["containerRun"], \
-                p["containerViews"], \
                 p["containerBase"], \
                 p["container2step"])
     def isContainer(self, name):
@@ -136,3 +150,10 @@ def getContainerImage(self,name):
         for p in self.yaml:
             if p["name"] == name:
                 return p["containerBase"]
+    def getContainer2base(self,name):
+        """
+        Brief: returns the image to be used in the second step of the Dockerfile
+        """
+        for p in self.yaml:
+            if p["name"] == name:
+                return p["container2base"]

fre/make/run_fremake_script.py

@@ -61,7 +61,7 @@ def compile_script_write_steps(yaml_obj,mkTemplate,src_dir,bld_dir,target,module
     compile_script = f"{bld_dir}/compile.sh" 
     return compile_script
 
-def dockerfile_write_steps(yaml_obj,makefile_obj,img,run_env,target,mkTemplate,td,cr,cb,cd):
+def dockerfile_write_steps(yaml_obj,makefile_obj,img,run_env,target,mkTemplate,s2i,td,cr,cb,cd):
     """
     Go through steps to create Dockerfile and container build script.
     """
@@ -70,7 +70,8 @@ def dockerfile_write_steps(yaml_obj,makefile_obj,img,run_env,target,mkTemplate,t
                                         libs = yaml_obj["container_addlibs"],
                                         RUNenv = run_env,
                                         target = target,
-                                        mkTemplate = mkTemplate)
+                                        mkTemplate = mkTemplate,
+                                        stage2base = s2i)
 
     dockerBuild.writeDockerfileCheckout("checkout.sh", td+"/checkout.sh")
     dockerBuild.writeDockerfileMakefile(makefile_obj.getTmpDir() + "/Makefile",
@@ -252,6 +253,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec
                 ## Run the checkout script
                 #          image="hpc-me-intel:2021.1.1"
                 image=modelYaml.platforms.getContainerImage(platformName)
+                stage2image = modelYaml.platforms.getContainer2base(platformName)
                 src_dir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/src"
                 bld_dir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/exec"
                 tmp_dir = "tmp/"+platformName
@@ -289,7 +291,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec
                 print("Makefile created here: " + tmp_dir + "/Makefile")# + "\n")
 
                 ## Build the dockerfile
-                #if is doesn't exist, write
+                # If is doesn't exist, write
                 curr_dir = os.getcwd()
                 if not os.path.exists(f"{curr_dir}/Dockerfile"):
                     print("Creating Dockerfile and build script...")
@@ -299,6 +301,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec
                                            run_env = platform["RUNenv"],
                                            target = target,
                                            mkTemplate = platform["mkTemplate"],
+                                           s2i = stage2image,
                                            td = tmp_dir,
                                            cr = platform["containerRun"],
                                            cb = platform["containerBuild"],
@@ -318,6 +321,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,exec
                                                target = target,
                                                td = tmp_dir,
                                                mkTemplate = platform["mkTemplate"],
+                                               s2i = stage2image,
                                                cr = platform["containerRun"],
                                                cb = platform["containerBuild"],
                                                cd = curr_dir)

fre/make/tests/null_example/platforms.yaml

@@ -18,3 +18,14 @@ platforms:
      compiler: gnu
      mkTemplate: /__w/fre-cli/fre-cli/mkmf/templates/linux-ubuntu-xenial-gnu.mk
      modelRoot: ${TEST_BUILD_DIR}/fremake_canopy/test
+   - name: con.twostep
+     compiler: intel
+     RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load [email protected]", "spack load [email protected]"]
+     modelRoot: /apps
+     container: True
+     containerBuild: "podman"
+     containerRun: "apptainer"
+     containerBase: "ecpe4s/noaa-intel-prototype:2023.09.25"
+     mkTemplate: "/apps/mkmf/templates/hpcme-intel21.mk"
+     container2step: True
+     container2base: "ecpe4s/noaa-intel-prototype:2023.09.25"

fre/make/tests/test_create_makefile.py

@@ -12,6 +12,7 @@
 YAMLFILE = "null_model.yaml"
 BM_PLATFORM = ["ncrc5.intel23"]
 CONTAINER_PLATFORM = ["hpcme.2023"]
+CONTAINER_PLAT2 = ["con.twostep"]
 TARGET = ["debug"]
 EXPERIMENT = "null_model_full"
 
@@ -73,3 +74,13 @@ def test_container_makefile_creation():
     create_makefile_script.makefile_create(yamlfile_path,CONTAINER_PLATFORM,TARGET)
 
     assert Path(f"tmp/{container_plat}/Makefile").exists()
+
+def test_container2step_makefile_creation():
+    """
+    Check the makefile is created when the two stage container is used
+    """
+    container_plat = CONTAINER_PLAT2[0]
+    yamlfile_path = f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}"
+    create_makefile_script.makefile_create(yamlfile_path,CONTAINER_PLAT2,TARGET)
+
+    assert Path(f"tmp/{container_plat}/Makefile").exists()

fre/make/tests/test_run_fremake.py

@@ -16,6 +16,7 @@
 YAMLPATH = f"{YAMLDIR}/{YAMLFILE}"
 PLATFORM = ["ci.gnu"]
 CONTAINER_PLATFORM = ["hpcme.2023"]
+CONTAINER_PLAT2 = ["con.twostep"]
 TARGET = ["debug"]
 BADOPT = ["foo"]
 EXPERIMENT = "null_model_full"
@@ -146,6 +147,33 @@ def test_run_fremake_run_script_creation_container():
     ''' checks (internal) container run script creation from previous test '''
     assert Path(f"tmp/{CONTAINER_PLATFORM[0]}/execrunscript.sh").exists()
 
+# tests container 2 stage build script/makefile/dockerfile creation
+def test_run_fremake_2stage_container():
+    '''run run-fremake with options for containerized build'''
+    run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLAT2, TARGET, False, 1, True, False, VERBOSE)
+
+def test_run_fremake_2stage_build_script_creation_container():
+    ''' checks container build script creation from previous test '''
+    assert Path("createContainer.sh").exists()
+
+def test_run_fremake_2stage_dockerfile_creation_container():
+    ''' checks dockerfile creation from previous test '''
+    assert Path("Dockerfile").exists()
+
+def test_run_fremake_2stage_checkout_script_creation_container():
+    ''' checks checkout script creation from previous test '''
+    assert Path(f"tmp/{CONTAINER_PLAT2[0]}/checkout.sh").exists()
+
+def test_run_fremake_2stage_makefile_creation_container():
+    ''' checks makefile creation from previous test '''
+    assert Path(f"tmp/{CONTAINER_PLAT2[0]}/Makefile").exists()
+
+def test_run_fremake_2stage_run_script_creation_container():
+    ''' checks (internal) container run script creation from previous test '''
+    assert Path(f"tmp/{CONTAINER_PLAT2[0]}/execrunscript.sh").exists()
+
+# tests for builds with multiple targets
+
 def test_run_fremake_container_force_checkout(capfd):
     '''run run-fremake with options for containerized build with force-checkout option'''
     run_fremake_script.fremake_run(YAMLPATH, CONTAINER_PLATFORM, TARGET, False, 1, True, False, VERBOSE, True, False, False)

fre/yamltools/tests/AM5_example/compile_yamls/platforms.yaml

@@ -3,24 +3,18 @@ platforms:
      compiler: intel
      modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
      modules: [ !join [*INTEL, "/2022.2.1"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
-     fc: ftn
-     cc: cc
      mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk"
      modelRoot: ${HOME}/fremake_canopy/test
    - name: ncrc5.intel23
      compiler: intel
      modulesInit: [" module use -a /ncrc/home2/fms/local/modulefiles \n","source $MODULESHOME/init/sh \n"]
      modules: [!join [*INTEL, "/2023.1.0"],"fre/bronx-20",cray-hdf5/1.12.2.3, cray-netcdf/4.9.0.3]
-     fc: ftn
-     cc: cc
      mkTemplate: "/ncrc/home2/fms/local/opt/fre-commands/bronx-20/site/ncrc5/$(INTEL).mk"
      modelRoot: ${HOME}/fremake_canopy/test
    - name: hpcme.2023
      compiler: intel
      RUNenv: [". /spack/share/spack/setup-env.sh", "spack load libyaml", "spack load [email protected]", "spack load [email protected]"]
      modelRoot: /apps
-     fc: mpiifort
-     cc: mpiicc
      container: True
      containerBuild: "podman"
      containerRun: "apptainer"