Update user manuals

.github/workflows/develop.yml

@@ -15,20 +15,27 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
-      # this is to fix GIT not liking owner of the checkout dir
+    # this is to fix GIT not liking owner of the checkout dir
     - name: Set ownership
       run: |
         chown -R $(id -u):$(id -g) $PWD
+
+    - name: Update pip
+      run: python3 -m pip install --upgrade pip
+
     - name: install gridlabd
       run: |
         autoreconf -isf
         ./configure
-        make system
+        make -j$(($(nproc)*3)) system
+
     - name: Install openfido
       run: curl -sL https://raw.githubusercontent.com/openfido/cli/main/install.sh | bash
+
     - name: Validate build
       run: |
         gridlabd -D keep_progress=TRUE -T 0 --validate -D github_actions=yes || ( utilities/save_validation_errors ; false )
+
     - name: Upload artifacts on failure
       uses: actions/upload-artifact@v2
       if: failure()

Makefile.am

@@ -350,7 +350,10 @@ troubleshooting-html:
 	@(cd $(top_srcdir) ; gawk -f utilities/troubleshooting.awk */*.cpp > $(top_srcdir)/documents/html/troubleshooting/index.html )
 
 documents/gridlabd.pdf: documents/gridlabd.glm
-	(cd documents ; gridlabd gridlabd.glm)
-	pandoc -V geometry:landscape documents/gridlabd.md -o documents/gridlabd.pdf
+	(cd documents ; $(DESTDIR)$(bindir)/gridlabd gridlabd.glm | echo "WARNING: unable to update documents/gridlabd.pdf")
+	test -f documents/gridlabd.md && pandoc -V geometry:landscape documents/gridlabd.md -o documents/gridlabd.pdf
 
-docs: $(docs_targets)
+docs: $(docs_targets) # documents/gridlabd.pdf
+
+docs_toc:
+	utilities/regen_toc >'docs/User manual/0 - Cover/1 - Contents.md'

cloud/websites/tutorials.gridlabd.us/library.csv

@@ -1,4 +1,8 @@
 User Tutorials
+user/quickstart,GridLAB-D Quickstart,0
+user/samplemodels,Accessing Sample Models,0
+user/weatherdata,Getting weather data,0
+user/loadmodels,Creating load models,0
 
 Developer Tutorials
 developer/session1,Introduction,27

configure.ac

@@ -622,9 +622,9 @@ if echo "$host_os" | grep '^linux' >/dev/null; then
         fi
 
         if test ! -d "$BINDIR/pkgenv/include/python*"; then
-            ln -s /usr/local/bin/python${PYEXE}-config $BINDIR/pkgenv/bin/python3-config
-            ln -s /usr/local/lib/*${PYEXE}* $BINDIR/pkgenv/lib
-            ln -s /usr/local/include/python${PYEXE} $BINDIR/pkgenv/include
+            ln -sF /usr/local/bin/python${PYEXE}-config $BINDIR/pkgenv/bin/python3-config
+            ln -sF /usr/local/lib/*${PYEXE}* $BINDIR/pkgenv/lib
+            ln -sF /usr/local/include/python${PYEXE} $BINDIR/pkgenv/include
         fi
 
     else
@@ -637,9 +637,9 @@ elif echo "$host_os" | grep '^darwin' >/dev/null; then
         # Add Intel Mac-specific configuration here
         /usr/local/bin/python$PYEXE -m venv "$BINDIR/pkgenv"
 
-        ln -s /usr/local/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/bin/python$PYEXE-config $BINDIR/pkgenv/bin/python3-config
-        ln -s /usr/local/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/include/python$PYEXE $BINDIR/pkgenv/include
-        ln -s /usr/local/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/lib/*${PYEXE}* $BINDIR/pkgenv/lib
+        ln -sF /usr/local/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/bin/python$PYEXE-config $BINDIR/pkgenv/bin/python3-config
+        ln -sF /usr/local/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/include/python$PYEXE $BINDIR/pkgenv/include
+        ln -sF /usr/local/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/lib/*${PYEXE}* $BINDIR/pkgenv/lib
 
         $PYDIR/python3 -m ensurepip --upgrade
         $PYDIR/python3 -m pip install matplotlib Pillow pandas numpy networkx pytz pysolar PyGithub scikit-learn xlrd boto3
@@ -651,9 +651,9 @@ elif echo "$host_os" | grep '^darwin' >/dev/null; then
         # Add arm64 Mac-specific configuration here
         /opt/homebrew/bin/python$PYEXE -m venv "$BINDIR/pkgenv"
 
-        ln -s /opt/homebrew/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/bin/python$PYEXE-config $BINDIR/pkgenv/bin/python3-config
-        ln -s /opt/homebrew/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/include/python$PYEXE $BINDIR/pkgenv/include
-        ln -s /opt/homebrew/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/lib/*${PYEXE}* $BINDIR/pkgenv/lib
+        ln -sF /opt/homebrew/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/bin/python$PYEXE-config $BINDIR/pkgenv/bin/python3-config
+        ln -sF /opt/homebrew/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/include/python$PYEXE $BINDIR/pkgenv/include
+        ln -sF /opt/homebrew/opt/python@$PYEXE/Frameworks/Python.framework/Versions/$PYEXE/lib/*${PYEXE}* $BINDIR/pkgenv/lib
 
         $PYDIR/python3 -m ensurepip --upgrade
         $PYDIR/python3 -m pip install matplotlib Pillow pandas numpy networkx pytz pysolar PyGithub scikit-learn xlrd boto3

converters/autotest/test_config.csv

@@ -0,0 +1,3 @@
+MY_INT,1
+MY_FLOAT,1.2
+MY_STRING,test string

converters/autotest/test_config.glm

@@ -0,0 +1,10 @@
+#ifexist ../test_config.csv
+#define DIR=..
+#endif
+
+#convert -i ${DIR:-.}/test_config.csv -o test_config_opt.glm -f csv-config -t glm-config
+#include "test_config_opt.glm"
+
+#ifexist ../test_config_opt.glm
+#on_exit 0 diff ../test_config_opt.glm test_config_opt.glm >gridlabd.diff
+#endif

converters/autotest/test_config_opt.glm

@@ -0,0 +1,3 @@
+#define MY_INT=1
+#define MY_FLOAT=1.2
+#define MY_STRING=test string

converters/autotest/test_library.csv

@@ -0,0 +1,3 @@
+class,name,value1,value2
+test,item1,1.2,3.4
+test,item2,5.6,7.8

converters/autotest/test_library.glm

@@ -0,0 +1,15 @@
+#ifexist ../test_library.csv
+#define DIR=..
+#endif
+
+class test {
+    double value1;
+    double value2;
+}
+
+#convert -i ${DIR:-.}/test_library.csv -o test_library_opt.glm -f csv-table -t glm-library
+#include "test_library_opt.glm"
+
+#ifexist ../test_library_opt.glm
+#on_exit 0 diff -I '^//' ../test_library_opt.glm test_library_opt.glm >gridlabd.diff
+#endif

converters/autotest/test_library_opt.glm

@@ -0,0 +1,13 @@
+// generated by csv-table2glm-library/convert('./test_library.csv','test_library_opt.glm',{})
+object test
+{
+	name "item1";
+	value1 "1.2";
+	value2 "3.4";
+}
+object test
+{
+	name "item2";
+	value1 "5.6";
+	value2 "7.8";
+}

converters/autotest/test_modify.csv

@@ -0,0 +1,4 @@
+load1.phases,ABC
+load1.nominal_voltage,120 V
+load1.constant_power_A,10
+load1.constant_power_B,8

converters/autotest/test_modify.glm

@@ -0,0 +1,14 @@
+module powerflow;
+object load {
+    name load1;
+}
+#ifexist ../test_modify.csv
+#define DIR=..
+#endif
+
+#convert -i ${DIR:-.}/test_modify.csv -o test_modify_opt.glm -f csv-modify -t glm-modify
+#include "test_modify_opt.glm"
+
+#ifexist ../test_modify_opt.glm
+#on_exit 0 diff ../test_modify_opt.glm test_modify_opt.glm >gridlabd.diff
+#endif

converters/autotest/test_modify_opt.glm

@@ -0,0 +1,4 @@
+modify load1.phases "ABC";
+modify load1.nominal_voltage "120 V";
+modify load1.constant_power_A "10";
+modify load1.constant_power_B "8";

converters/autotest/test_object.csv

@@ -0,0 +1,3 @@
+name,value1,value2
+item1,1.2,3.4
+item2,5.6,7.8

converters/autotest/test_object.glm

@@ -0,0 +1,15 @@
+#ifexist ../test_object.csv
+#define DIR=..
+#endif
+
+class test {
+    double value1;
+    double value2;
+}
+
+#convert -i ${DIR:-.}/test_object.csv -o test_object_opt.glm -f csv-table -t glm-object class=test
+#include "test_object_opt.glm"
+
+#ifexist ../test_object_opt.glm
+#on_exit 0 diff -I '^//' ../test_object_opt.glm test_object_opt.glm >gridlabd.diff
+#endif

converters/autotest/test_object_opt.glm

@@ -0,0 +1,11 @@
+// /usr/local/opt/gridlabd/4.3.1-220814-develop_user_manual/share/gridlabd/csv-table2glm-object.py ./test_object.csv test_object_opt.glm {} 
+object test { 
+	name "item1";
+	value1 1.2;
+	value2 3.4;
+}
+object test { 
+	name "item2";
+	value1 5.6;
+	value2 7.8;
+}

converters/autotest/test_weather.csv

@@ -0,0 +1,22 @@
+timestamp,temperature
+2022-01-01 00:00:00 PST,+44.422
+2022-01-01 01:00:00 PST,+42.262
+2022-01-01 02:00:00 PST,+42.442
+2022-01-01 03:00:00 PST,+42.622
+2022-01-01 04:00:00 PST,+44.602
+2022-01-01 05:00:00 PST,+41.002
+2022-01-01 06:00:00 PST,+42.802
+2022-01-01 08:00:00 PST,+41.002
+2022-01-01 09:00:00 PST,+44.602
+2022-01-01 10:00:00 PST,+46.402
+2022-01-01 11:00:00 PST,+50.002
+2022-01-01 12:00:00 PST,+57.202
+2022-01-01 13:00:00 PST,+64.402
+2022-01-01 17:00:00 PST,+62.602
+2022-01-01 18:00:00 PST,+53.602
+2022-01-01 19:00:00 PST,+51.802
+2022-01-01 20:00:00 PST,+48.202
+2022-01-01 21:00:00 PST,+46.402
+2022-01-01 22:00:00 PST,+44.602
+2022-01-01 23:00:00 PST,+42.802
+2022-01-02 00:00:00 PST,+44.602

converters/autotest/test_weather.glm

@@ -0,0 +1,18 @@
+clock {
+    timezone "PST+8PDT";
+    starttime "2022-01-01 00:00:00 PST";
+    stoptime "2022-01-02 00:00:00 PST";
+}
+#weather get CA-Twentynine_Palms.tmy3
+#input "CA-Twentynine_Palms.tmy3"
+module tape { csv_header_type NAME; }
+object recorder {
+    parent ${FIND class=climate};
+    file test_weather.csv;
+    interval -1;
+    property temperature;
+}
+
+#ifexist ../test_weather.csv
+#on_exit 0 diff ../test_weather.csv test_weather.csv > gridlabd.diff
+#endif

converters/csv-table2glm-library.py

@@ -89,6 +89,7 @@ class specified, which must be a class defined in the module specified.
         for id, data in table.iterrows():
             if 'class' in data.keys():
                 oclass = data['class']
+                del data['class']
             elif 'class' in options.keys():
                 oclass = options['class']
             else:

converters/csv-table2glm-object.py

@@ -57,4 +57,4 @@ def convert (input_name, output_name=None, options={} ) :
 						if not key in ["module","class"]:
 							glm.write(f"\t{key} \"{value}\";\n")
 				glm.write("}\n")
-		glm.close()
+		glm.close()

docs/Developer/Class/Create.md

@@ -0,0 +1 @@
+TODO

docs/Developer/Module/Create.md

@@ -0,0 +1 @@
+TODO

docs/Developer/Validation.md

@@ -0,0 +1,99 @@
+[[/Develop/Validation]] - Validation Guide
+
+GridLAB-D validation subsystem is designed to detect code and modeling errors, and identify unexpected changes in the outputs from simulations.  Validation is run as part of the build automation. In addition the `--validate` option can be used to run validation tests locally.
+
+The validation system scan the source tree for all folders named `autotest`.  In any autotest folders, all the files name `test_*.glm` are processed as follows:
+
+1. A subfolder is created using the GLM file's base name
+
+2. The GLM files is copied to the subfolder.
+
+3. The GLM file is run with the command line `gridlabd BASENAME.glm --redirect=all OPTIONS` where OPTIONS are all the options given after the `--validate` command line option.
+
+4. The exit status of the simulation is interpreted as follows
+
+  - If the BASENAME includes the string `_err`, then the exit code must be positive and less than 128 for the validation test to pass, i.e., an error occurred.
+
+  - If the BASENAME includes the string `_exc`, then the exit code must be strictly less than zero or greater than 128, i.e., an exception occurred.
+
+  - If the BASENAME includes the string `_opt`, then the exit status is ignored.
+
+  - Otherwise, the exit status must be exactly zero, i.e., the simulation completed normally.
+
+A report of the validation test results is written to the file `validation.txt`.  Progress followed by a summary is output to the screen.
+
+## Frequently used options
+
+There are a few very useful command line that be used *before* the `--validate` option.
+
+- `-T COUNT` or `--threadcount COUNT`: This options enable parallel processing of the validation tests.  The count `0` indicates the validation system should try to use the maximum number of available processors.  A positive number may be given to indicate the exact number of processors to use. This can reduce the time needed to perform validate tests.
+
+- `-W FOLDER`: This options is used to limit in which folder the search for `autotest` folders is performed. This is useful to perform quick tests of only particular parts of the entire `gridlabd` system.
+
+- `-D keep_progress=TRUE`: This option prevent overwrite of previous progress output. This is useful when performing automated builds where the progress output needs to be preserved.
+
+# Validation Conventions
+
+Validation test emerge from two primary activities.  The first is code development, during test cases are identified to ensure that the code works as intended.  The second is from issues identified by users, when test cases are used to exemplify a problem or scenario that might not work as expected. 
+
+A number of conventions have evolved in validation testing as a result of these activites.
+
+## Use simple models
+
+Some tests need to use very complex models, but most tests should be as simple as possible.
+
+## Use fast tests
+
+Minimize the testing time. There is no need to run a 1 year simulation when a 1 day simulation will suffice.
+
+## Test for edge cases and invalid inputs
+
+Don't just test for what works. Also test for what unusual and what doesn't work.  Make sure you test for strange cases that give unintuitive results, and for bad inputs such out of range values, incompatible units, or utter nonsense. Be creative.
+
+## Check the results
+
+Autotests should use `assert` object or `#on_exit` macros to verify that the results are correct.  Asserts can be used to check values during a simulation, and `#on_exit` can be used to check after completion.
+
+The most common `assert` is simple to very that a value does not go outside a reasonable range, e.g.,
+
+~~~
+object house
+{
+    object assert
+    {
+        target "air_temperature";
+        relation "inside";
+        lower 60;
+        upper 90;
+    };
+}
+~~~
+
+which causes the simulation to fail if the target value goes outside the specified range.
+
+The most common use of the `#on_exit` macro is to compare an output file to a reference output file, e.g.,
+
+~~~
+#define OUTPUT=${modelname/.glm/.csv}
+module tape
+{
+    csv_header_type NAME;
+}
+module residential;
+object house
+{
+    object recorder
+    {
+        file ${OUTPUT};
+    };
+}
+#ifexist "../${OUTPUT}"
+#on_exit 0 diff -w ../${OUTPUT} ${OUTPUT}
+#endif
+~~~
+
+which causes the simulation to fail is the recorder output differs from the one provided in the parent autotest folder.  This reference file is automatically generated by the simulation when it is run from the parent folder, so the developer must remember to create and update it whenever the code or model changes in a significant way.
+
+# Automation artifacts
+
+GridLAB-D's automated build system runs validation every time the code is updated.  If the validation test fails, an artifact file is created that can be downloaded.  The artifact file will contain the autotest folder contents created by the validation tests that failed.  These can be examined to determine the cause of the validation test failure.

docs/GLM/Directive/Dump.md

@@ -13,3 +13,7 @@ dump interval filename;
 The `dump` directive causes the dump file `filename` to be generated at the `interval` times. 
 
 If the filename starts with a dash, then the output is sent to stdout in `GLM` format.  Adding the extension after the dash causes the specified format to use generated (i.e., `glm` or `json`).
+
+# See also
+
+* [[/Global/Filesave_options]]