From b713446ee1c73d6d3496de01500ed08d78f9b04e Mon Sep 17 00:00:00 2001
From: Guillaume <glipsl@ipsl.fr>
Date: Thu, 12 Oct 2017 14:54:04 +0200
Subject: [PATCH] add CDS variable event

---
 .idea/emacs.xml        |   6 ++
 .idea/misc.xml         |   4 +
 .idea/modules.xml      |   8 ++
 .idea/synda.iml        |  12 +++
 .idea/workspace.xml    | 175 +++++++++++++++++++++++++++++++++++++++++
 sdp/bin/spconst.py     |   6 +-
 sdt/bin/sdconst.py     |   2 +
 sdt/bin/sdstat.py      |  65 ++++++++-------
 sdt/bin/sdsubparser.py |   2 +-
 sdt/bin/sdtiaction.py  |  50 ++++++++++++
 10 files changed, 294 insertions(+), 36 deletions(-)
 create mode 100644 .idea/emacs.xml
 create mode 100644 .idea/misc.xml
 create mode 100644 .idea/modules.xml
 create mode 100644 .idea/synda.iml
 create mode 100644 .idea/workspace.xml

diff --git a/.idea/emacs.xml b/.idea/emacs.xml
new file mode 100644
index 00000000..a1e816fc
--- /dev/null
+++ b/.idea/emacs.xml
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="EmacsSettings">
+    <option name="emacsPath" value="/usr/bin/emacs" />
+  </component>
+</project>
\ No newline at end of file
diff --git a/.idea/misc.xml b/.idea/misc.xml
new file mode 100644
index 00000000..75213d35
--- /dev/null
+++ b/.idea/misc.xml
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 2.7.13 (~/anaconda/bin/python)" project-jdk-type="Python SDK" />
+</project>
\ No newline at end of file
diff --git a/.idea/modules.xml b/.idea/modules.xml
new file mode 100644
index 00000000..7fb29f42
--- /dev/null
+++ b/.idea/modules.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/synda.iml" filepath="$PROJECT_DIR$/.idea/synda.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file
diff --git a/.idea/synda.iml b/.idea/synda.iml
new file mode 100644
index 00000000..6f63a63c
--- /dev/null
+++ b/.idea/synda.iml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="projectConfiguration" value="Nosetests" />
+    <option name="PROJECT_TEST_RUNNER" value="Nosetests" />
+  </component>
+</module>
\ No newline at end of file
diff --git a/.idea/workspace.xml b/.idea/workspace.xml
new file mode 100644
index 00000000..7e057056
--- /dev/null
+++ b/.idea/workspace.xml
@@ -0,0 +1,175 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ChangeListManager">
+    <list default="true" id="95c0cb97-dd14-4b07-aade-b9c5d58a54bb" name="Default" comment="">
+      <change type="MODIFICATION" beforePath="$PROJECT_DIR$/sdt/bin/sdtiaction.py" afterPath="$PROJECT_DIR$/sdt/bin/sdtiaction.py" />
+    </list>
+    <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
+    <option name="TRACKING_ENABLED" value="true" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="FileEditorManager">
+    <leaf SIDE_TABS_SIZE_LIMIT_KEY="300" />
+  </component>
+  <component name="Git.Settings">
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
+  </component>
+  <component name="IdeDocumentHistory">
+    <option name="CHANGED_PATHS">
+      <list>
+        <option value="$PROJECT_DIR$/sdt/bin/sdtiaction.py" />
+        <option value="$PROJECT_DIR$/sdt/bin/sdsubparser.py" />
+        <option value="$PROJECT_DIR$/sdt/bin/sdconst.py" />
+        <option value="$PROJECT_DIR$/sdp/bin/spconst.py" />
+        <option value="$PROJECT_DIR$/sdt/bin/sdstat.py" />
+      </list>
+    </option>
+  </component>
+  <component name="ProjectFrameBounds" extendedState="6">
+    <option name="y" value="22" />
+    <option name="width" value="1080" />
+    <option name="height" value="1893" />
+  </component>
+  <component name="ProjectLevelVcsManager" settingsEditedManually="true" />
+  <component name="ProjectView">
+    <navigator currentView="ProjectPane" proportions="" version="1">
+      <flattenPackages />
+      <showMembers />
+      <showModules />
+      <showLibraryContents />
+      <hideEmptyPackages />
+      <abbreviatePackageNames />
+      <autoscrollToSource />
+      <autoscrollFromSource />
+      <sortByType />
+      <manualOrder />
+      <foldersAlwaysOnTop value="true" />
+    </navigator>
+    <panes>
+      <pane id="ProjectPane">
+        <subPane>
+          <expand>
+            <path>
+              <item name="synda" type="b2602c69:ProjectViewProjectNode" />
+              <item name="synda" type="462c0819:PsiDirectoryNode" />
+            </path>
+          </expand>
+          <select />
+        </subPane>
+      </pane>
+      <pane id="Scope" />
+      <pane id="Scratches" />
+    </panes>
+  </component>
+  <component name="PropertiesComponent">
+    <property name="settings.editor.selected.configurable" value="project.propVCSSupport.Mappings" />
+    <property name="last_opened_file_path" value="$PROJECT_DIR$" />
+  </component>
+  <component name="RunDashboard">
+    <option name="ruleStates">
+      <list>
+        <RuleState>
+          <option name="name" value="ConfigurationTypeDashboardGroupingRule" />
+        </RuleState>
+        <RuleState>
+          <option name="name" value="StatusDashboardGroupingRule" />
+        </RuleState>
+      </list>
+    </option>
+  </component>
+  <component name="ShelveChangesManager" show_recycled="false">
+    <option name="remove_strategy" value="false" />
+  </component>
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="95c0cb97-dd14-4b07-aade-b9c5d58a54bb" name="Default" comment="" />
+      <created>1507801097570</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1507801097570</updated>
+    </task>
+    <servers />
+  </component>
+  <component name="ToolWindowManager">
+    <frame x="0" y="22" width="1080" height="1894" extended-state="6" />
+    <layout>
+      <window_info id="Project" active="true" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.25433525" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" />
+      <window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" />
+      <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" />
+      <window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="false" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" />
+      <window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
+      <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" />
+      <window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" />
+      <window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
+      <window_info id="Data View" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" />
+      <window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" />
+      <window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
+      <window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" />
+      <window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" />
+      <window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" />
+      <window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
+      <window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" />
+    </layout>
+  </component>
+  <component name="VcsContentAnnotationSettings">
+    <option name="myLimit" value="2678400000" />
+  </component>
+  <component name="VcsManagerConfiguration">
+    <ignored-roots>
+      <path value="$PROJECT_DIR$" />
+    </ignored-roots>
+  </component>
+  <component name="XDebuggerManager">
+    <breakpoint-manager />
+    <watches-manager />
+  </component>
+  <component name="editorHistoryManager">
+    <entry file="file://$PROJECT_DIR$/sdt/bin/sdstat.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="660">
+          <caret line="44" column="42" lean-forward="false" selection-start-line="44" selection-start-column="42" selection-end-line="44" selection-end-column="42" />
+          <folding>
+            <element signature="e#704#714#0" expanded="true" />
+          </folding>
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/sdp/bin/spconst.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="885">
+          <caret line="59" column="21" lean-forward="true" selection-start-line="59" selection-start-column="21" selection-end-line="59" selection-end-column="21" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/sdt/bin/sdconst.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="15">
+          <caret line="35" column="0" lean-forward="false" selection-start-line="35" selection-start-column="0" selection-end-line="35" selection-end-column="0" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/sdt/bin/sdsubparser.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="1046">
+          <caret line="194" column="17" lean-forward="false" selection-start-line="194" selection-start-column="17" selection-end-line="194" selection-end-column="17" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+    <entry file="file://$PROJECT_DIR$/sdt/bin/sdtiaction.py">
+      <provider selected="true" editor-type-id="text-editor">
+        <state relative-caret-position="0">
+          <caret line="565" column="20" lean-forward="false" selection-start-line="565" selection-start-column="20" selection-end-line="565" selection-end-column="20" />
+          <folding />
+        </state>
+      </provider>
+    </entry>
+  </component>
+</project>
\ No newline at end of file
diff --git a/sdp/bin/spconst.py b/sdp/bin/spconst.py
index 8677b409..2e198bd1 100755
--- a/sdp/bin/spconst.py
+++ b/sdp/bin/spconst.py
@@ -30,6 +30,8 @@
 EVENT_CDF_COR_VARIABLE_O='cdf_cor_variable_o' # project with One variable per dataset
 EVENT_CDF_COR_VARIABLE_N='cdf_cor_variable_n' # project with N variable per dataset
 
+EVENT_CDS_VARIABLE='cds_variable'
+
 EVENT_STATUS_NEW='new'
 EVENT_STATUS_ANOMALY='anomaly'
 EVENT_STATUS_OLD='old'
@@ -57,8 +59,8 @@
 #
 DEFAULT_PRIORITY=1000
 #
-AUTHORIZED_PROJECT=['CORDEX','CMIP5','CMIP6']
-PROJECT_WITH_ONE_VARIABLE_PER_DATASET=['CORDEX','CMIP6']
+AUTHORIZED_PROJECT=['CORDEX','CMIP5','CMIP6','c3scmip5','c3scordex']
+PROJECT_WITH_ONE_VARIABLE_PER_DATASET=['CORDEX','CMIP6','c3scmip5','c3scordex']
 #
 TRIGGER_TYPE_NV2D='NV2D' # N 'variable pipeline' trigger 'dataset pipeline'
 TRIGGER_TYPE_V2V='V2V'   # 'variable pipeline' triggers 'variable pipeline'
diff --git a/sdt/bin/sdconst.py b/sdt/bin/sdconst.py
index cff42450..366a8879 100755
--- a/sdt/bin/sdconst.py
+++ b/sdt/bin/sdconst.py
@@ -32,6 +32,8 @@
 EVENT_CDF_COR_VARIABLE_O='cdf_cor_variable_o' # project with One variable per dataset
 EVENT_CDF_COR_VARIABLE_N='cdf_cor_variable_n' # project with N variable per dataset
 
+EVENT_CDS_VARIABLE='cds_variable'
+
 EVENT_STATUS_NEW='new'
 EVENT_STATUS_ANOMALY='anomaly'
 EVENT_STATUS_OLD='old'
diff --git a/sdt/bin/sdstat.py b/sdt/bin/sdstat.py
index a9641a0f..3ce5d6f4 100755
--- a/sdt/bin/sdstat.py
+++ b/sdt/bin/sdstat.py
@@ -23,17 +23,16 @@
 import argparse
 import json
 import humanize
-import sdapp
-import sdconst
 import sdstatutils
 import syndautils
 import sdexception
 import sdpipelineprocessing
+from sdtools import print_stderr
 
-def run(args):
 
+def run(args):
     try:
-        metadata=syndautils.file_full_search(args)
+        metadata = syndautils.file_full_search(args)
     except sdexception.EmptySelectionException, e:
         print_stderr("You must specify at least one facet to perform this action.")
         return 1
@@ -41,46 +40,46 @@ def run(args):
     if args.dry_run:
         return 0
 
+    statuses = sdstatutils.init_table()
+    po = sdpipelineprocessing.ProcessingObject(sdstatutils.get_statuses, statuses)
+    sdpipelineprocessing.run_pipeline(metadata, po)
 
-    statuses=sdstatutils.init_table()
-    po=sdpipelineprocessing.ProcessingObject(sdstatutils.get_statuses,statuses)
-    sdpipelineprocessing.run_pipeline(metadata,po)
-
-
-    total=sdstatutils.get_total(statuses)
-    print_summary(statuses,total)
+    total = sdstatutils.get_total(statuses)
+    print_summary(statuses, total)
 
     return 0
 
-def print_summary(statuses,total,mode='line'):
-    if mode=='delimited':
+
+def print_summary(statuses, total, mode='line'):
+    if mode == 'delimited':
         print "Total size;Pending size;Local size;Total files count;Pending files count;Local files count"
-        print "%s;%s;%s;%s;%s;%s"%( humanize.naturalsize(total['all']['size'],gnu=False),
-                                    humanize.naturalsize(total['pending']['size'],gnu=False),
-                                    humanize.naturalsize(statuses['done']['size'],gnu=False),
-                                    humanize.naturalsize(statuses['new']['size'],gnu=False),
-                                    total['all']['count'],
-                                    total['pending']['count'],
-                                    statuses['done']['count'],
-                                    statuses['new']['count'])
-    elif mode=='line':
-        print "Total files count: %s"%total['all']['count']
-        #print "Pending files count: %s"%total['pending']['count']
+        print "%s;%s;%s;%s;%s;%s" % (humanize.naturalsize(total['all']['size'], gnu=False),
+                                     humanize.naturalsize(total['pending']['size'], gnu=False),
+                                     humanize.naturalsize(statuses['done']['size'], gnu=False),
+                                     humanize.naturalsize(statuses['new']['size'], gnu=False),
+                                     total['all']['count'],
+                                     total['pending']['count'],
+                                     statuses['done']['count'],
+                                     statuses['new']['count'])
+    elif mode == 'line':
+        print "Total files count: %s" % total['all']['count']
+        # print "Pending files count: %s"%total['pending']['count']
         for s in statuses.keys():
-            count=statuses[s]['count']
-            if count>0:
-                print "%s files count: %s"%(s.title(),count)
-        print "Total size: %s"%humanize.naturalsize(total['all']['size'],gnu=False)
-        #print "Pending size: %s"%humanize.naturalsize(total['pending']['size'],gnu=False)
+            count = statuses[s]['count']
+            if count > 0:
+                print "%s files count: %s" % (s.title(), count)
+        print "Total size: %s" % humanize.naturalsize(total['all']['size'], gnu=False)
+        # print "Pending size: %s"%humanize.naturalsize(total['pending']['size'],gnu=False)
         for s in statuses.keys():
-            size=statuses[s]['size']
-            if size>0:
-                print "%s files size: %s"%(s.title(),humanize.naturalsize(size,gnu=False))
+            size = statuses[s]['size']
+            if size > 0:
+                print "%s files size: %s" % (s.title(), humanize.naturalsize(size, gnu=False))
+
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
     args = parser.parse_args()
 
-    files=json.load( sys.stdin ) # warning: load list in memory
+    files = json.load(sys.stdin)  # warning: load list in memory
 
     run(files)
diff --git a/sdt/bin/sdsubparser.py b/sdt/bin/sdsubparser.py
index 1d11fa00..4d2a45b5 100644
--- a/sdt/bin/sdsubparser.py
+++ b/sdt/bin/sdsubparser.py
@@ -218,7 +218,7 @@ def run(subparsers):
     subparser.add_argument('-c','--columns',type=int,default=1)
 
     subparser=create_subparser(subparsers,'pexec',help='Execute post-processing task')
-    subparser.add_argument('order_name',choices=['cdf'],help='Order name')
+    subparser.add_argument('order_name',choices=['cdf','cds'],help='Order name')
 
     subparser=create_subparser(subparsers,'queue',common_option=False,help='Display download queue status',example=sdcliex.queue())
     subparser.add_argument('project',nargs='?',default=None,help='ESGF project (e.g. CMIP5)')
diff --git a/sdt/bin/sdtiaction.py b/sdt/bin/sdtiaction.py
index f2e10227..887be466 100755
--- a/sdt/bin/sdtiaction.py
+++ b/sdt/bin/sdtiaction.py
@@ -601,6 +601,56 @@ def pexec(args):
                 print_stderr("Post-processing task successfully submitted (order_dataset_count=%d,order_variable_count=%d)"%(order_dataset_count,order_variable_count))
         else:
             print_stderr('Data not found')
+
+    elif args.order_name=='cds':
+        selection_filename = None
+
+        # use search-api operator to build datasets list
+        stream = syndautils.get_stream(subcommand=args.subcommand, selection_file=args.selection_file, no_default=args.no_default)
+        sddeferredbefore.add_forced_parameter(stream, 'type', 'Dataset')
+
+        dataset_found_count = 0
+        order_variable_count = 0
+        for facets_group in stream:  # we need to process each facets_group one by one because of TAG45345JK3J53K
+
+            metadata = sdsearch.run(stream=[facets_group], post_pipeline_mode='dataset')  # TAGJ43KJ234JK
+
+            dataset_found_count += metadata.count()
+
+            if metadata.count() > 0:
+
+                # WART
+                # (gets overwritten at each iteration, but not a big deal as always the same value)
+                if selection_filename is None:  # this is to keep the first found value (i.e. if last facets_group is empty but not the previous ones do not keep the last one (which would be None))
+
+                    dataset = metadata.get_one_file()
+                    selection_filename = sdpostpipelineutils.get_attached_parameter__global([dataset], 'selection_filename')  # note that if no files are found at all for this selection (no matter the status), then the filename will be blank
+
+                for d in metadata.get_files():  # warning: load list in memory
+                    if d['status'] == sdconst.DATASET_STATUS_COMPLETE:
+
+                        # TAG45J4K45JK
+
+                        # send cds variable order
+                        # (note: total number of variable event is given by: "total+=#variable for each ds")
+                        for v in d['variable']:
+                            if v in facets_group['variable']:  # TAG45345JK3J53K (we check here that the variable has been asked for in the first place)
+                                order_variable_count += 1
+                                sdpporder.submit(sdconst.EVENT_CDS_VARIABLE, d['project'], d['model'], d['local_path'], variable=v, commit=False)
+
+        sddb.conn.commit()
+
+        if dataset_found_count > 0:
+            if order_variable_count == 0:
+                print_stderr("Data not ready (data must be already downloaded before performing pexec task): operation cancelled")
+            else:
+                sdhistorydao.add_history_line(sdconst.ACTION_PEXEC, selection_filename)
+
+                print_stderr(
+                    "Post-processing task successfully submitted (order_variable_count=%d)" % (order_variable_count))
+        else:
+            print_stderr('Data not found')
+
     else:
         print_stderr("Invalid order name ('%s')"%args.order_name)
         return 1