Merge tag 'v3.00.55' into stable

stable v3.00.55

.github/workflows/flow.yml

@@ -32,7 +32,7 @@ jobs:
       fail-fast: false
       matrix:
        which_test: [ static_flow, no_mirror, flakey_broker, dynamic_flow, restart_server, partitioned_flow ]
-       osver: [ "ubuntu-20.04", "ubuntu-22.04" ]
+       osver: [ "ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04" ]
 
     runs-on: ${{ matrix.osver }}
 

.github/workflows/flow_amqp_consumer.yml

@@ -29,7 +29,7 @@ jobs:
       fail-fast: false
       matrix:
        which_test: [ static_flow, no_mirror, flakey_broker, dynamic_flow, restart_server ]
-       osver: [ "ubuntu-20.04", "ubuntu-22.04" ]
+       osver: [ "ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04" ]
 
     runs-on: ${{ matrix.osver }}
 

.github/workflows/flow_basic.yml

@@ -26,7 +26,7 @@ jobs:
       # Don't cancel the entire matrix when one job fails
       fail-fast: false
       matrix:
-       osver: [ "ubuntu-20.04", "ubuntu-22.04" ]
+       osver: [ "ubuntu-20.04", "ubuntu-22.04", "ubuntu-24.04" ]
 
     runs-on: ${{ matrix.osver }}
 

.github/workflows/flow_mqtt.yml

@@ -29,7 +29,7 @@ jobs:
       fail-fast: false
       matrix:
        which_test: [ static_flow, no_mirror, flakey_broker, dynamic_flow ]
-       osver: [ "ubuntu-22.04" ]
+       osver: [ "ubuntu-22.04", "ubuntu-24.04" ]
 
     runs-on: ${{ matrix.osver }}
 

.github/workflows/flow_redis.yml

@@ -29,7 +29,7 @@ jobs:
       fail-fast: false
       matrix:
        which_test: [ static_flow, no_mirror, flakey_broker, dynamic_flow, restart_server ]
-       osver: [ "ubuntu-22.04" ]
+       osver: [ "ubuntu-22.04", "ubuntu-24.04" ]
 
     runs-on: ${{ matrix.osver }}
 

debian/changelog

@@ -1,3 +1,27 @@
+metpx-sr3 (3.00.55) unstable; urgency=medium
+
+  * fix #1219 correct date conversion.
+  * fix #1214 scheduling refactor addresses a regression found in rc1
+  * fix #953 sr3 status queue size updates again.
+  * fix #1157 transfer/stat() call improves behaviour with overly dynamic data.
+  * fix #1209 Added *schedule_time* to scheduled flows (and polls)
+  * fix #1159 provide obvious error message when no configurations matched.
+  * fix #1166 better download decision criteria (arbitrary identity vs. file size)
+  * fix #1151 calculate on download not working.
+  * fix #1173 sr3 stop improvements.
+  * fix #1208 badly expiring timers (crashing mostly senders.)
+  * fix #1150 bc transport api refactor.
+  * fix #1147 documenting messageAgeMax, post_messageAgeMax
+  * fix #1155 ack refactor (PR #1174)
+  * fix #1172 clash in names (unit tests.)
+  * fix #1195 more but still concise logging.
+  * fix #1192 bug in rejectPercent. improved lag display.
+  * fix #1204 display processes still running during each wait of stop.
+  * fix #1217 one minute of wiggle room when scheduled flows come due.
+  * fix #1218 permissions of directories created by senders. 
+
+ -- SSC-5CD2310S60 <[email protected]>  Tue, 23 Sep 2024 09:47:36 -0400
+
 metpx-sr3 (3.00.54post2) unstable; urgency=medium
 
   * 3.00.54post1 package generation on launchpad failed, re-uploading as post2
@@ -28,21 +52,21 @@ metpx-sr3 (3.00.54) unstable; urgency=medium
   * fix #1097 message rates < 1 were parsed as zero, now work correctly.
   * messages reviewed & consolidated: #1094, #1099 (log de-cluttering.)
   * fix crashes/problems with statehost #1076, #1087, #1096
-  * when plugins go bad, report better, recover better: #1085, #1091, 
+  * when plugins go bad, report better, recover better: #1085, #1091,
   * performance improvements #1083, #1086
   * fix #1079 watch posting events it should not.
   * fix #1076 bogus messages and mkdir crash on HPC ( #1087 )
   * merge PR #1067 (closing #824 ) sr3 status flow state detail improved.
     * adds lag,rtry,slow,reje states to status display.
     * adds checks for running process to cleanup, fail if running.
     * adds cleanup to remove, don't remove if cleanup fails.
-    * adds progressive logs to transfers  (closing #966) 
+    * adds progressive logs to transfers  (closing #966)
   * several fixes for sender crashes ( #1091, #1095, #1099 )
   * add #1054 can now convert multiple configs, and overwrite (with --wololo)
   * fixed #1064 poll crash.
   * fixed #927 sanity not restarting crashed polls.
   * reduced processing overhead ( #1086 )
-  * misc bug fixes ( #1070, #1092, #1103, #1108, #1109 ) 
+  * misc bug fixes ( #1070, #1092, #1103, #1108, #1109 )
   * NEW added logFormat option.
   * bug fixes and unit tests for AM ( #1036, $1068, #1074, #1075, #1078, #1079 )
   * many unit tests added, improved coverage (though still poor.)

docs/source/Contribution/Release.rst

@@ -55,12 +55,14 @@ To publish a pre-release one needs to:
   - find redhat 8 server.  build package::
    
          * git checkout development_py36
+         * git pull
          * python3 setup.py bdist_rpm*
          * run flow tests
 
-  - find redhat 9 server,  build package:: 
+  - find redhat 9 server,  build package::
 
          * git checkout development_py36
+         * git pull
          * python3 setup.py bdist_rpm*
          * run flow tests
 
@@ -77,31 +79,32 @@ To publish a pre-release one needs to:
 
      * git pull
      * git checkout development_py36
+     * git pull
      * git tag -a o3.xx.yyrcz -m "pre-release o3.xx.yy.rcz"
      * git pull 
      * git checkout pre_release_py36
      * git pull
      * git merge development_py36
      * git push
+     * git push origin o3.xx.yyrcz
 
      * git checkout development
      * git tag -a v3.xx.yy.rcZ -m "pre-release v3.xx.yy.rcz"
      * git checkout pre_release
      * git pull
      * git merge development
      * git push
+     * git push origin v3.xx.yyrcz
 
-- push the above.  
 
-     * git push origin o3.xx.yyrcz
-     * git push origin v3.xx.yyrcz
 
 - pypi.org
 
   - to ensure compatiblity with python3.6, update a python3.6 branch (for redhat 8 and/or ubuntu 18.)
   - use the python3.6 branch to release to pypi (because upward compatibility works, but not downward.)
 
     * git checkout pre_release_py36
+    * git pull
     * python3 setup.py bdist_wheel
 
   - upload the pre-release so that installation with pip succeeds.
@@ -131,16 +134,17 @@ To publish a pre-release one needs to:
 
   - find redhat 8 server. build package:: 
 
-        git checkout stable_py36
+        git checkout pre-release_py36
+        git pull
         python3 setup.py bdist_rpm 
 
 
   - find redhat 9 server, build package::
 
-        git checkout stable_py36
+        git checkout pre-release_py36
+        git pull
         python3 setup.py bdist_rpm 
 
-
 - on github: Draft a release.
 
   - create release notes as prompted.
@@ -163,26 +167,50 @@ the stable release does not require any explicit testing.
 
 * merge from pre-release to stable::
 
+    
+     git checkout pre-release
+     git pull
      git checkout stable
+     git pull
      git merge pre-release
+     git push
+
      # there will be conflicts here for debian/changelog and sarracenia/_version.py
      # for changelog:
      #   - merge all the rcX changelogs into a single stable one.
      #   - ensure the version at the top is correct and tagged 'unstable'
      #   - edit the signature at the bottom for reflect who you are, and current date.
-     # for sarracenia/_version.py
+     # sr3 for sarracenia/_version.py (v2 sarra/__init__.py )
      #   - fix it so it shows the correct stable version.
      git tag -a v3.xx.yy -m "v3.xx.yy"
      git push origin v3.xx.yy
 
 * merge from pre-release_py36 to stable_py36::
 
+     git checkout pre_release_py36
+     git pull
      git checkout stable_py36
+     git pull
      git merge pre_release_py36
+     git push
      # same editing required as above.
      git tag -a o3.xx.yy -m "o3.xx.yy"
      git push origin o3.xx.yy
 
+* pypi.org
+
+  - to ensure compatiblity with python3.6, update a python3.6 branch (for redhat 8 and/or ubuntu 18.)
+  - use the python3.6 branch to release to pypi (because upward compatibility works, but not downward.)
+
+    * git checkout stable_py36
+    * git pull
+    * python3 setup.py bdist_wheel
+
+  - upload the pre-release so that installation with pip succeeds.
+
+    * twine upload dist/the_wheel_produced_above.whl 
+
+
 * go on Launchpad, 
 
    * stable branch ready.
@@ -197,6 +225,7 @@ the stable release does not require any explicit testing.
 * go on ubuntu 18.04, build bdist_wheel::
 
       git checkout stable_py36
+      git pull
       python3 setup.py bdist_wheel 
 
 note that *pip3 install wheel* is needed, because the one from
@@ -205,12 +234,18 @@ ubuntu 18 is not compatible with the current pypi.org.
 * go on redhat 8, build rpm::
 
       git checkout stable_py36
+      git pull
       python3 setup.py bdist_rpm 
+      ls dist/
+      mv rpm file to have rh8 in the name somewhere.
 
-* go on redhat 9, build rpm::
+* go on redhat 9, build rpm **NOTE: Broken!, for now use redhat 8 process** ::
 
       git checkout stable_py36
+      git pull
       rpmbuild --build-in-place -bb metpx-sr3.spec
+      ls dist/
+      mv rpm file to have rh8 in the name somewhere.
 
 
 * On github.com, create release.
@@ -219,8 +254,7 @@ ubuntu 18 is not compatible with the current pypi.org.
   * attach wheel build on ubuntu 18.
   * attach redhat 8 rpm
   * attach redhat 9 rpm 
-  * attach windows exe
-
+  * attach windows exe ... see: `Building a Windows Installer`_
 
 Details
 -------
@@ -246,11 +280,13 @@ prior to accepting a release, and barring known exceptions,
   tests: static, no_mirror, flakey_broker, restart_server, dynamic_flow::
 
        git checkout pre_release_py36
+       git pull
        python3 setup.py bdist_rpm
  
 * Redhat 9 rpms currently do not work... vm and run the flow test there to ensure that it works::
 
        git checkout pre_release_py36
+       git pull
        python3 setup.py bdist_rpm
          
 
@@ -316,7 +352,7 @@ This is done to *start* development on a version. It should be done on developme
 after every release.
 
 * git checkout development
-* Edit ``sarracenia/_version.py`` manually and set the version number.
+* Edit ``sarracenia/_version.py`` (``sarra/__init__.py`` for v2) manually and set the version number.
 * Edit CHANGES.rst to add a section for the version.
 * run dch to start the changelog for the current version. 
   * change *unstable* to *UNRELEASED* (maybe done automatically by dch.)
@@ -655,13 +691,16 @@ by pynsist to build the executable, so look at::
     https://www.python.org/downloads/windows/
 
 Then go look on python.org, for the "right" version (for 3.10, it is 3.10.11 )
-Then, from the shell, run::
+It will contain *embed* in the file names. Once you find the correct version
+From the shell, run::
 
- sudo apt install nsis
- pip3 install pynsist wheel
- ./generate-win-installer.sh 3.10.11 2>&1 > log.txt
+   sudo apt install nsis
+   pip3 install pynsist wheel
+   ./generate-win-installer.sh 3.10.11 2>&1 > log.txt
 
-The final package will be generated into build/nsis directory.
+The final package will be generated into *build/nsis* directory. Sometimes editing of 
+the *generate-win-installer.sh* script is needed to add *--no-isolation* to the *python -m build* line.
+It's not clear when that is needed.
 
 
 github

docs/source/Explanation/CommandLineGuide.rst

@@ -954,8 +954,13 @@ to poll the remote resources every thirty minutes. Alternatively::
    scheduled_hour 1,13,19
    scheduled_minute 27
 
-specifies that poll be run at 1:27, 13:27, and 19:27 each day.
-
+specifies that poll be run at 1:27, 13:27, and 19:27 each day. Alternatively::
+
+   scheduled_time 15:30,20:30,23:15
+
+will poll the remote resources at exactly 15:30, 16:30 and 23:15.
+
+
 By default, sr_poll sends its post notification message to the broker with default exchange
 (the prefix *xs_* followed by the broker username). The *post_broker* is mandatory.
 It can be given incomplete if it is well defined in the credentials.conf file.