Ltarchive updates

cime_config/cesm/archive.xml

@@ -345,7 +345,7 @@
 	<subdir>hist</subdir> 
 	<keep_last_in_rundir>false</keep_last_in_rundir>
       </file_extension>
-      <file_extension suffix=".h\.[\w\-]*\.nc$">      
+      <file_extension suffix=".h\.[\w\-\.]*\.nc$">      
 	<subdir>hist</subdir> 
 	<keep_last_in_rundir>false</keep_last_in_rundir>
       </file_extension>

cime_config/cesm/machines/config_lt_archive.xml

@@ -16,10 +16,4 @@
     <machine name="pleiades-san">
       <lt_archive_args>--mode copy_dirs_ssh --ssh_loc lfe --rm_loc_files TRUE</lt_archive_args>
     </machine>
-    <machine name="hobart">
-      <lt_archive_args></lt_archive_args>
-    </machine>
-    <machine name="yellowstone">
-      <lt_archive_args>--mode copy_files</lt_archive_args>
-    </machine>
 </config_lt_archive>

cime_config/cesm/machines/template.ltarchive → cime_config/cesm/machines/template.lt_archive

@@ -43,6 +43,64 @@ my $logger = Log::Log4perl::get_logger();
 # global data needed by the script, stuff like the max number of threads,
 # -------------------------------------------------------------------------
 
+#-----------------------------------------------------------------------------------------------
+# checkRun - check if run is complete
+#-----------------------------------------------------------------------------------------------
+sub checkRun
+{
+    my $statusFile = shift;
+    my $runComplete = 0;
+
+    $logger->debug("In checkRun..."); 
+#
+# check if the run completed successfully
+#
+    if( -f $statusFile ) 
+    {
+	open my $CASESTATUS, "<", "$statusFile"  or die qq(lt_archive: unable to open $statusFile. Exiting...\n);
+	while( <$CASESTATUS> ) 
+	{
+	    chomp $_;
+	        if( /^run SUCCESSFUL/ )
+		{
+		    $runComplete = 1;
+		}
+	}
+	close( $CASESTATUS );
+    }
+}
+
+#-----------------------------------------------------------------------------------------------
+# checkSta - check if short term archive is complete
+#-----------------------------------------------------------------------------------------------
+sub checkSta
+{
+    my $statusFile = shift;
+    my $staComplete = 0;
+
+    $logger->debug("In checkSta..."); 
+#
+# check if the short term archive completed successfully
+#
+    if( -f $statusFile ) 
+    {
+	open my $STASTATUS, "<", "$statusFile"  or die qq(lt_archive: unable to open $statusFile. Exiting...\n);
+	while( <$STASTATUS> ) 
+	{
+	    chomp $_;
+	        if( /^st_archive process complete/ )
+		{
+		    $staComplete = 1;
+		}
+	}
+	close( $STASTATUS );
+    }
+}
+
+
+#-----------------------------------------------------------------------------------------------
+# main
+#-----------------------------------------------------------------------------------------------
 sub main
 {
     $ENV{'maxthrds'} = 1;
@@ -71,7 +129,6 @@ sub main
 	$ENV{DOUT_S_ROOT} = $config{'DOUT_S_ROOT'};
 	$ENV{DOUT_L_MSROOT} = $config{'DOUT_L_MSROOT'};
 	$ENV{DOUT_L_HPSS_ACCNT} = $config{'DOUT_L_HPSS_ACCNT'};
-	$ENV{DOUT_L_SAVE_ALL_ON_DISK} = $config{'DOUT_L_SAVE_ALL_ON_DISK'};
 
 	my $LID = strftime("%y%m%d-%H%M%S", localtime);
 
@@ -93,14 +150,18 @@ sub main
     resubmitCheck();
 }
 
+#-----------------------------------------------------------------------------------------------
+# resubmitCheck - check if the case needs to be resubmitted
+#-----------------------------------------------------------------------------------------------
+
 sub resubmitCheck()
 {
     if($config{RESUBMIT} > 0)
     {
         my $submitscript = "$config{CASEROOT}/$config{CASE}.submit";
         my $cwd = getcwd;
         chdir $config{CASEROOT};
-        my $resubmitcomand = "$submitscript -resubmit -scriptname $config{'CASE'}.lt_archive";
+        my $resubmitcommand = "$submitscript -resubmit -scriptname $config{'CASE'}.lt_archive";
         $logger->debug(" running resubmit check $resubmitcommand");
         if(-e $testlog)
         {

driver_cpl/cime_config/config_component.xml

@@ -432,25 +432,30 @@
     We will not document this further in this guide.</desc>
   </entry> 
 
-  <entry id="DOUT_S_SAVE_EVERY_NTH_RESTART_FILE_SET"> 
-    <type>integer</type>
-    <default_value>0</default_value>
+  <entry id="DOUT_S_SAVE_DATES_RESTART_FILE_SET"> 
+    <type>string</type>
+    <default_value>all</default_value>
     <group>run_data_archive</group>
     <file>env_run.xml</file>
-    <desc>Save every Nth set of restart files and delete all others
-      If value is greater than 0, then only save the nth restart set in the $DOUT_S_ROOT/$CASE/rest location. 
-      Always preserve the most recent restart set regardless.</desc>
+    <desc>Comma separated List with dates of the month to save restart file sets in the $DOUT_S_ROOT/$CASE/rest directory.
+      If one of the list elements is last, then it saves the last day of the month where the date is 28,29,30 or 31 
+      depending on the month. For example, if the list contains 10,20,last, then the corresponding restart sets
+      saved in $DOUT_S_ROOT/$CASE/rest are 2016-01-10-00000, 2016-01-20-00000, 2016-01-31-00000 for January and
+      2016-02-10-00000, 2016-02-20-00000, 2016-02-29-00000 for February due to 2016 being a leap year. 
+      If the list is set to none, then all restart sets are deleted.
+      If the list is set to all (default), then all restart sets are saved.
+      This variable is used by the clobber_archive script to selectively delete files in the short-term archive locations.</desc>
   </entry> 
 
-  <entry id="DOUT_S_SAVE_ALL_ON_DISK"> 
+  <entry id="DOUT_S_CREATE_LINKED_ARCHIVE"> 
     <type>logical</type>
     <valid_values>TRUE,FALSE</valid_values>
     <default_value>TRUE</default_value>
     <group>run_data_archive</group>
     <file>env_run.xml</file>
-    <desc>logical to save contents of the short term archive on disk
-      If TRUE, create hardlinks from the short term archive $DOUT_S_ROOT.locked/$CASE
-      directory to the $DOUT_S_ROOT/$CASE directory.</desc>
+    <desc>Logical to save contents of the short term archive on disk.
+      If TRUE, create hardlinks from the short term archive $DOUT_S_ROOT/archive.locked/$CASE
+      directory to the $DOUT_S_ROOT/archive/$CASE directory.</desc>
   </entry> 
 
   <entry id="DOUT_L_MS"> 
@@ -471,16 +476,6 @@
     <desc></desc>
   </entry>
 
-  <entry id="DOUT_L_SAVE_ALL_ON_DISK"> 
-    <type>logical</type>
-    <valid_values>TRUE,FALSE</valid_values>
-    <default_value>TRUE</default_value>
-    <group>run_data_archive</group>
-    <file>env_run.xml</file>
-    <desc>Save contents of short term archive on disk after the long term archiver has successfully completed
-    If TRUE, this keeps the output data in DOUT_S_ROOT on disk after the long term (lt_archive.sh) archiver is run.</desc>
-  </entry> 
-
   <!-- ===================================================================== -->
   <!-- definitions machines specific -->
   <!-- ===================================================================== -->

scripts/Tools/case.setup

@@ -280,12 +280,12 @@ if (! $clean ) {
 	$logger->info("Creating batch script case.st_archive");
 	$batchmaker->overrideNodeCount(1);
 	$batchmaker->set({job=>'st_archive'});
-	$batchmaker->makeBatchScript("$xmlvars{'MACHDIR'}/template.starchive", "$xmlvars{'CASEROOT'}/case.st_archive");
+	$batchmaker->makeBatchScript("$xmlvars{'MACHDIR'}/template.st_archive", "$xmlvars{'CASEROOT'}/case.st_archive");
 
 	$logger->info("Creating batch script case.lt_archive");
 	$batchmaker->overrideNodeCount(1);
 	$batchmaker->set({job=>'lt_archive'});
-	$batchmaker->makeBatchScript("$xmlvars{'MACHDIR'}/template.ltarchive", "$xmlvars{'CASEROOT'}/case.lt_archive");
+	$batchmaker->makeBatchScript("$xmlvars{'MACHDIR'}/template.lt_archive", "$xmlvars{'CASEROOT'}/case.lt_archive");
 
 	# Make a copy of env_mach_pes.xml in order to be able 
 	# to check that it does not change once case.setup is invoked

scripts/Tools/lt_archive.sh

@@ -182,24 +182,14 @@ if [ "$mode" == "copy_dirs_hsi" ]; then
 	exit -1
     fi
 
-    # Long-term archiver for HPSS (Trey White, December 6, 2011)
     date
 
-    if [ ! $?DOUT_L_HPSS_ACCNT ]; then
-	DOUT_L_HPSS_ACCNT=0
-    fi
-
-    # check if files on st_archive should be saved or not
-    saveFlag="-dPR"
-    if [ $DOUT_L_SAVE_ALL_ON_DISK == "TRUE" ] ; then
-	saveFlag="-PR"
-    fi
-
-    # send files to HPSS and delete upon success
+    # send files to HPSS
+    saveFlag="-PR"
     cd $DOUT_S_ROOT
     hsiArgs="mkdir -p $DOUT_L_MSROOT ; chmod +t $DOUT_L_MSROOT ; cd $DOUT_L_MSROOT ; put $saveFlag *"
     # echo $hsiArgs
-    if [ $DOUT_L_HPSS_ACCNT -gt 0 ]; then
+    if ! [[ "$DOUT_L_HPSS_ACCNT" =~ "0000*" ]]; then  
 	hsi -a $DOUT_L_HPSS_ACCNT "$hsiArgs"
     else
 	hsi "$hsiArgs"
@@ -317,8 +307,6 @@ if [ "$mode" == "copy_dirs_local" ]; then
 			rm -f ${file}
 		    else
 			echo "local file and long-term archive file are NOT the same size... ${file} will remain on local disk"
-			#exit -1 #??? ask francis if this is right 
-                        # Not sure what to do here... maybe make the log entry and carry on...
 		    fi
 		fi
 	    done # for file