diff --git a/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0001.log b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0001.log new file mode 100644 index 00000000..81f7289e --- /dev/null +++ b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0001.log @@ -0,0 +1,9 @@ +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 110.5 | 361.59 | 410.76 | 217.4 | 52.41 +Coll | 22.174 | 69.358 | 105.6 | 95.0 | 10.05 +Sort | 48.822 | 156.12 | 198.1 | 146.5 | 22.63 +Comm | 0.57662 | 0.74641 | 1.2112 | 15.3 | 0.11 +Modify | 0.044491 | 0.14381 | 0.67954 | 40.0 | 0.02 +Output | 0.19404 | 1.0017 | 7.2883 | 105.4 | 0.15 +Other | | 101 | | | 14.64 diff --git a/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0008.log b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0008.log new file mode 100644 index 00000000..b7646707 --- /dev/null +++ b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0008.log @@ -0,0 +1,9 @@ +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 135.6 | 382.41 | 407.35 | 101.6 | 54.34 +Coll | 27.647 | 70.87 | 126.3 | 37.3 | 10.07 +Sort | 59.573 | 157.56 | 165.23 | 56.4 | 22.39 +Comm | 0.98096 | 1.1726 | 2.0141 | 9.0 | 0.17 +Modify | 0.042252 | 0.093733 | 0.63089 | 33.9 | 0.01 +Output | 0.28671 | 1.2288 | 7.131 | 43.4 | 0.17 +Other | | 90.45 | | | 12.85 diff --git a/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0016.log b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0016.log new file mode 100644 index 00000000..74f5f8ac --- /dev/null +++ b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0016.log @@ -0,0 +1,9 @@ +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 151.1 | 377.83 | 405.9 | 87.9 | 52.18 +Coll | 31.182 | 69.201 | 121.61 | 26.2 | 9.56 +Sort | 60.038 | 156.78 | 165.19 | 45.9 | 21.65 +Comm | 1.0515 | 1.2436 | 2.1974 | 8.0 | 0.17 +Modify | 0.042161 | 0.080089 | 0.65554 | 31.4 | 0.01 +Output | 0.50613 | 1.4353 | 6.8989 | 36.2 | 0.20 +Other | | 117.6 | | | 16.24 diff --git a/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0032.log b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0032.log new file mode 100644 index 00000000..90ca259e --- /dev/null +++ b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0032.log @@ -0,0 +1,9 @@ +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 140.17 | 376.51 | 407.67 | 78.9 | 50.59 +Coll | 36.525 | 69.102 | 120.45 | 20.8 | 9.28 +Sort | 62.203 | 156.44 | 164.56 | 38.6 | 21.02 +Comm | 1.1024 | 1.3148 | 2.1336 | 5.9 | 0.18 +Modify | 0.040976 | 0.071406 | 0.62065 | 28.1 | 0.01 +Output | 0.5231 | 1.5368 | 7.0738 | 30.6 | 0.21 +Other | | 139.3 | | | 18.72 diff --git a/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0064.log b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0064.log new file mode 100644 index 00000000..00d7bdd4 --- /dev/null +++ b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0064.log @@ -0,0 +1 @@ +WIP diff --git a/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0128.log b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0128.log new file mode 100644 index 00000000..21dcaf61 --- /dev/null +++ b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0128.log @@ -0,0 +1,9 @@ +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 84.649 | 368.77 | 448.56 | 71.2 | 46.96 +Coll | 23.61 | 67.034 | 103.47 | 14.6 | 8.54 +Sort | 35.737 | 154.7 | 163.88 | 34.1 | 19.70 +Comm | 2.0262 | 2.2258 | 2.9132 | 4.0 | 0.28 +Modify | 0.038906 | 0.062956 | 0.63114 | 21.2 | 0.01 +Output | 0.89732 | 1.9417 | 8.5356 | 24.3 | 0.25 +Other | | 190.5 | | | 24.26 diff --git a/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0256.log b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0256.log new file mode 100644 index 00000000..4b407baf --- /dev/null +++ b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0256.log @@ -0,0 +1,9 @@ +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 45.807 | 369.55 | 433.94 | 69.7 | 46.02 +Coll | 9.5725 | 66.926 | 120.55 | 15.8 | 8.33 +Sort | 18.124 | 155.66 | 168.11 | 32.4 | 19.38 +Comm | 2.3658 | 2.557 | 3.1786 | 3.4 | 0.32 +Modify | 0.03839 | 0.060081 | 0.62783 | 18.1 | 0.01 +Output | 1.1599 | 2.1722 | 10.311 | 24.9 | 0.27 +Other | | 206.2 | | | 25.67 diff --git a/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0512.log b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0512.log new file mode 100644 index 00000000..00d7bdd4 --- /dev/null +++ b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-0512.log @@ -0,0 +1 @@ +WIP diff --git a/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-1024.log b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-1024.log new file mode 100644 index 00000000..a7aa4840 --- /dev/null +++ b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-1024.log @@ -0,0 +1,9 @@ +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 99.221 | 366.16 | 474.26 | 68.7 | 44.64 +Coll | 22.305 | 66.098 | 104.74 | 13.8 | 8.06 +Sort | 45.493 | 155.32 | 196.93 | 31.1 | 18.93 +Comm | 3.2565 | 3.4645 | 4.2795 | 3.0 | 0.42 +Modify | 0.034187 | 0.052975 | 0.66257 | 13.5 | 0.01 +Output | 1.3687 | 2.4426 | 9.1743 | 22.9 | 0.30 +Other | | 226.7 | | | 27.64 diff --git a/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-2048.log b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-2048.log new file mode 100644 index 00000000..71bfcc9a --- /dev/null +++ b/doc/sphinx/08_sparta/ats3--scale--breakdown--nodes-2048.log @@ -0,0 +1,9 @@ +Section | min time | avg time | max time |%varavg| %total +--------------------------------------------------------------- +Move | 38.678 | 364.46 | 483.63 | 69.0 | 43.35 +Coll | 9.682 | 65.913 | 114.89 | 13.7 | 7.84 +Sort | 16.133 | 155.13 | 198.36 | 31.8 | 18.45 +Comm | 10.207 | 10.415 | 11.252 | 1.7 | 1.24 +Modify | 0.035347 | 0.053417 | 0.71562 | 11.6 | 0.01 +Output | 0.88878 | 2.7248 | 10.728 | 19.6 | 0.32 +Other | | 242 | | | 28.78 diff --git a/doc/sphinx/08_sparta/sparta.rst b/doc/sphinx/08_sparta/sparta.rst index a8e2139a..87b1a9e4 100644 --- a/doc/sphinx/08_sparta/sparta.rst +++ b/doc/sphinx/08_sparta/sparta.rst @@ -657,6 +657,115 @@ per node. SPARTA Multi-Node Weak Scaling Throughput on Crossroads with ppc=35 +Timing Breakdown +^^^^^^^^^^^^^^^^ + +Timing breakdown information directly from SPARTA is provided for +various node counts. SPARTA writes out a timer block that resembles +the following. + +.. code-block:: + + Section | min time | avg time | max time |%varavg| %total + --------------------------------------------------------------- + Move | 110.5 | 361.59 | 410.76 | 217.4 | 52.41 + Coll | 22.174 | 69.358 | 105.6 | 95.0 | 10.05 + Sort | 48.822 | 156.12 | 198.1 | 146.5 | 22.63 + Comm | 0.57662 | 0.74641 | 1.2112 | 15.3 | 0.11 + Modify | 0.044491 | 0.14381 | 0.67954 | 40.0 | 0.02 + Output | 0.19404 | 1.0017 | 7.2883 | 105.4 | 0.15 + Other | | 101 | | | 14.64 + +A desription of the work performed for each of the sections is +provided below. + +``Move`` + Particle advection through the mesh, i.e., particle push + +``Coll`` + Particle collisions + +``Sort`` + Particle sorting (i.e., make a list of all particles in each grid + cell) and reorder (i.e., reorder the particle array by grid cell) + +``Comm`` + The bulk of the MPI communications + +``Modify`` + Time spent in diagnostics like "fixes" or "computes" + +``Output`` + Time spent writing statistical output to log, or other, file(s) + +``Other`` + Leftover time not captured by the categories above; this can + include load imbalance (i.e., ranks waiting at a collective + operation) + +These tables are provided below for the various rank counts for +reference. + + +1 Node +"""""" + +.. literalinclude:: ats3--scale--breakdown--nodes-0001.log + + +8 Nodes +""""""" + +.. literalinclude:: ats3--scale--breakdown--nodes-0008.log + + +16 Nodes +"""""""" + +.. literalinclude:: ats3--scale--breakdown--nodes-0016.log + + +32 Nodes +"""""""" + +.. literalinclude:: ats3--scale--breakdown--nodes-0032.log + + +64 Nodes +"""""""" + +.. literalinclude:: ats3--scale--breakdown--nodes-0064.log + + +128 Nodes +""""""""" + +.. literalinclude:: ats3--scale--breakdown--nodes-0128.log + + +256 Nodes +""""""""" + +.. literalinclude:: ats3--scale--breakdown--nodes-0256.log + + +512 Nodes +""""""""" + +.. literalinclude:: ats3--scale--breakdown--nodes-0512.log + + +1024 Nodes +"""""""""" + +.. literalinclude:: ats3--scale--breakdown--nodes-1024.log + + +2048 Nodes +"""""""""" + +.. literalinclude:: ats3--scale--breakdown--nodes-2048.log + References ==========