Skip to content

Commit

Permalink
v2.21
Browse files Browse the repository at this point in the history 
Former-commit-id: b064a1821c6607906cf8e1ec5d765fb0f0ef09ca [formerly 1f734b10db1adf06fbe54c9bde29aab515fd96e0] [formerly 8f41754f7ba927b97eefb32a281c956f4a1f5325 [formerly cd04e31b5b21e93963833ee56fda8ce2084865a4]]
Former-commit-id: 5eb93b0ffcde6ce4082a308b6d4c40c94b42e7e0 [formerly de4dc96c1a4fce3b414cabc84d154c7757811dbb]
Former-commit-id: 2e328bffa1ddcef6948e261189924642a02b45fe
Former-commit-id: 722977b77578c5dc4a1863c36dd37263063d0c79
  • Loading branch information
Bob Simons committed Oct 9, 2022
1 parent 54a0239 commit a041dda
Show file tree
Hide file tree
Showing 11 changed files with 142 additions and 75 deletions.
8 changes: 4 additions & 4 deletions WEB-INF/classes/gov/noaa/pfel/coastwatch/TestAll.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ public static void main(String args[]) throws Throwable {
// String tFileName = "/u00/satellite/MH1/chla/1day/AQUA_MODIS.20220301.L3m.DAY.CHL.chlor_a.4km.NRT.nc";
// String2.log(NcHelper.ncdump(tFileName, "-h"));

// DasDds.main(new String[]{"nesdisVHNnoaaSNPPnoaa20chlaDaily", "-verbose"});
// DasDds.main(new String[]{"esrlNcepRe", "-verbose"});

// String2.log(EDDTableFromAsciiFiles.generateDatasetsXml("S://obisSubset/", ".*\\.csv",
// "", "", 1, 2, ",", 1000000000, "", "", "", "", "", "", "myInfo", "myInstitution", "mySummary", "myTitle",
Expand Down Expand Up @@ -529,15 +529,15 @@ public static void main(String args[]) throws Throwable {
// String2.log(NcHelper.ncdump("/u00/data/points/caricoos/181p1_historic.nc", "-v metaStationLatitude;metaStationLongitude"));
/*
s = EDDTableFromMultidimNcFiles.generateDatasetsXml(
"/data/sarah/", ".*\\.nc", "",
"TIME_PRE", //dims
"/data/bodc/", "Cabot.*\\.nc", "",
"", //dims
1440,
"", "", "", "", //pre, post, extract, varname
true, //removeMVRows //often true
"", //sort files by profile_time
"", "", "", "",
0, //standardizeWhat 1+2(numericTime)+256(catch numeric mv)+4096(units)
"DEPTH,TIME_PRE", //treatDimensionsAs
"", //treatDimensionsAs
"", //cacheFromUrl /catalog.html
null) + "\n";
String2.setClipboardString(s); String2.log(s);
Expand Down
4 changes: 3 additions & 1 deletion WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ public class EDStatic {
* <br>2.18 released on 2022-02-23
* <br>2.19 released on 2022-09-10
* <br>2.20 released on 2022-09-30
* <br>2.21 released on 2022-10-09
*
* For master branch releases, this will be a floating point
* number with 2 decimal digits, with no additional text.
Expand All @@ -200,7 +201,7 @@ public class EDStatic {
* A request to http.../erddap/version will return just the number (as text).
* A request to http.../erddap/version_string will return the full string.
*/
public static String erddapVersion = "2.20"; //see comment above
public static String erddapVersion = "2.21"; //see comment above

/**
* This is almost always false.
Expand Down Expand Up @@ -4598,6 +4599,7 @@ public static void addIntroStatistics(StringBuilder sb) {
}
} catch (Exception e) {
}
sb.append("Number of active requests=" + activeRequests.size() + "\n");
}

/**
Expand Down
13 changes: 11 additions & 2 deletions WEB-INF/classes/gov/noaa/pfel/erddap/util/EmailThread.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,8 @@ public class EmailThread extends Thread {
private long lastStartTime = -1; //-1 if session not active
public long lastSessionMillis = -1; //duration

public static int sleepMillis = 5000;
public static long defaultSleepMillis = 15000;
public static long sleepMillis = defaultSleepMillis;


/**
Expand All @@ -59,7 +60,7 @@ public long elapsedTime() {


/**
* This repeatedly: sleeps for 5 seconds, then sends all pending emails.
* This repeatedly: sleeps for sleepMillis, then sends all pending emails.
*/
public void run() {

Expand Down Expand Up @@ -154,6 +155,9 @@ public void run() {
}
}

//reset sleepMillis because openEmailSession succeeded
sleepMillis = defaultSleepMillis;

String2.log("%%% EmailThread session finished after email #" + (EDStatic.nextEmail - 1) +
" at " + Calendar2.getCurrentISODateTimeStringLocalTZ());

Expand All @@ -170,6 +174,11 @@ public void run() {
" at " + Calendar2.getCurrentISODateTimeStringLocalTZ() + "\n" +
MustBe.throwableToString(e));

//openEmailSession and other failures: wait longer before try again to avoid e.g.,
// "jakarta.mail.AuthenticationFailedException: 454 4.7.0 Too many login attempts, please try again later."
if (sleepMillis * 2 < 5 * Calendar2.MILLIS_PER_MINUTE)
sleepMillis *= 2;

} finally {
lastStartTime = -1;
try {
Expand Down
2 changes: 1 addition & 1 deletion download/AccessToPrivateDatasets.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ <h2><a class="selfLink" id="contact" href="#contact" rel="bookmark">Contact</a><

<br>&nbsp;
<hr>
<p>ERDDAP, Version 2.20
<p>ERDDAP, Version 2.21
<br><a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html">Disclaimers</a> |
<a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html#privacyPolicy">Privacy Policy</a>
<p>&nbsp;
Expand Down
2 changes: 1 addition & 1 deletion download/EDDTableFromEML.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -606,7 +606,7 @@ <h2><a class="selfLink" id="contact" href="#contact" rel="bookmark">Contact</a><

<br>&nbsp;
<hr>
<p>ERDDAP, Version 2.20
<p>ERDDAP, Version 2.21
<br><a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html">Disclaimers</a> |
<a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html#privacyPolicy">Privacy Policy</a>

Expand Down
2 changes: 1 addition & 1 deletion download/NCCSV.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -930,7 +930,7 @@ <h2><a class="selfLink" id="contact" href="#contact" rel="bookmark">Contact</a><

<br>&nbsp;
<hr>
<p>ERDDAP, Version 2.20
<p>ERDDAP, Version 2.21
<br><a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html">Disclaimers</a> |
<a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html#privacyPolicy">Privacy Policy</a>
</div>
Expand Down
2 changes: 1 addition & 1 deletion download/NCCSV_1.00.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -859,7 +859,7 @@ <h2><a class="selfLink" id="contact" href="#contact" rel="bookmark">Contact</a><

<br>&nbsp;
<hr>
<p>ERDDAP, Version 2.20
<p>ERDDAP, Version 2.21
<br><a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html">Disclaimers</a> |
<a rel="bookmark" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html#privacyPolicy">Privacy Policy</a>
</div>
Expand Down
69 changes: 34 additions & 35 deletions download/changes.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,49 +72,41 @@ <h1 style="text-align:center;">ERDDAP Changes</h1>


<h2><a class="selfLink" id="changes2.20" href="#changes2.20" rel="bookmark">Changes</a>
in ERDDAP version 2.20 (released 2022-09-30)</h2>
in ERDDAP version 2.21 (released 2022-10-09)</h2>
<ul>
<li><strong>New Features and Changes (for users):</strong>
<ul>
<li>(None)
<br>&nbsp;
</ul>
<li><strong>Things ERDDAP Administrators Need to Know and Do:</strong>
<ul>
<li>TO DO: In [tomcat]/conf/context.xml, right before <kbd>&lt;/Context&gt;</kbd>,
change the <kbd>Resources</kbd> tag (or add it if it isn't already there) to
increase the <kbd>cacheMaxSize</kbd> setting from 80000 (or whatever it was) to 200000:
<br><kbd>&lt;Resources cachingAllowed="true" cacheMaxSize="200000" /&gt;</kbd>
<br>This avoids numerous warnings in catalina.out that all start with
<br><kbd>"WARNING [main] org.apache.catalina.webresources.Cache.getResource Unable to add the resource at [/WEB-INF/classes/..."</kbd>
<ul>
<li>BUG FIX: Sometimes, the new email system attempted to log in too often,
which caused Google Email servers to reject all future log in attempts. Now,
the email system avoids this and related problems.
<br>&nbsp;

</ul>
</ul>

<!-- ****************************************************************************** -->


<h2><a class="selfLink" id="changes2.20" href="#changes2.20" rel="bookmark">Changes</a>
in ERDDAP version 2.20 (released 2022-09-30)</h2>
<ul>
<li><strong>New Features and Changes (for users):</strong>
<ul>
<li>(None)
<br>&nbsp;
</ul>
<li><strong>Things ERDDAP Administrators Need to Know and Do:</strong>
<ul>
<li>IMPROVED: We re-enabled the old memory management system (Math2.ensureMemoryAvailable) and modified
the new memory management system (EDStatic.shedThisRequest) to work better with it.
<p>ERDDAP shouldn't ever crash or freeze up. If it does, one of the
most likely causes is insufficient memory. You can monitor memory usage
by looking at the status.html web page, which now includes a line like
<br><kbd>0 gc calls, 0 requests shed, and 0 dangerousMemoryEmails since last major LoadDatasets</kbd>
<br>(those are progressively more serious events)
<br>and a <kbd>gc Calls</kbd> column in the table of statistics.
You can tell how memory-stressed your ERDDAP is by watching these numbers
in addition to the <kbd>MB inUse</kbd> columns.
Higher numbers indicate more stress.
<ul>
<li><kbd>MB inUse</kbd> should always be less than half of your Xmx memory setting.
Larger numbers are serious trouble.
<li><kbd>gc calls</kbd> indicates the number of times ERDDAP called the
garbage collector to try to alleviate high memory usage.
<li><kbd>shed</kbd> indicates the number of incoming requests that
were shed (with HTTP error number 503, Service Unavailable) because
memory use was already too high. Ideally, no requests should be shed.
<li><kbd>dangerousMemoryEmails</kbd> -
If memory use becomes dangerously high, ERDDAP sends an email to the email addresses listed in
emailEverythingToCSV (in setup.xml) with a list of the active user requests. As the email says,
please forward these emails to erd.data at noaa.gov so we can use the information
to improve future versions of ERDDAP.
See <a rel="help"
href="https://coastwatch.pfeg.noaa.gov/erddap/download/setup.html#MemoryStatus"
>Memory Status</a> for details.
<br>&nbsp;
</ul>

<li>CHANGED: The default for &lt;ipAddressMaxRequests&gt; in datasets.xml
was increased from 7 to 15. It's clear that some legitimate WMS clients
Expand Down Expand Up @@ -159,6 +151,12 @@ <h2><a class="selfLink" id="changes2.19" href="#changes2.19" rel="bookmark">Chan
There are no other changes that you need to make to your ERDDAP
installation related to this change. In other words, ERDDAP works
as it did before.

<p>Don't forget to make the ERDDAP-related changes to Tomcat's server.xml and context.xml
when you upgrade Tomcat. See ERDDAP's
<a rel="help" href="https://coastwatch.pfeg.noaa.gov/erddap/download/setup.html#tomcat"
>Tomcat installation instructions</a>.

<p>My impression of Java 17 is that it prefers more processing power and memory
for long-running, larger applications like ERDDAP, so it works slightly slower
than Java 8 with low power computers (e.g., 2 cores and minimal RAM)
Expand All @@ -168,10 +166,10 @@ <h2><a class="selfLink" id="changes2.19" href="#changes2.19" rel="bookmark">Chan
<a rel="bookmark" href="https://www.howtogeek.com/668986/how-to-use-the-linux-top-command-and-understand-its-output/">top<img
src="../images/external.png" alt=" (external link)"
title="This link to an external website does not constitute an endorsement."></a>
to check resource usage and consider giving ERDDAP more resources.
to check resource usage and consider giving ERDDAP more resources, notably more memory.
Memory is cheap! Most phones have more processors and memory than
the servers that some of you are using to run ERDDAP!
Thanks to Erin Turnbull.
<br>Thanks to Erin Turnbull.
<br>&nbsp;
<li>TO DO: If you use ERDDAP to access Cassandra, for Cassandra, you need to keep using the version
of Java that you were using for running the Cassandra.
Expand All @@ -181,6 +179,7 @@ <h2><a class="selfLink" id="changes2.19" href="#changes2.19" rel="bookmark">Chan
consider changing to these settings in your datasets.xml file:<kbd>
<br>&nbsp;&nbsp;&lt;nGridThreads&gt;3&lt;/nGridThreads&gt;
<br>&nbsp;&nbsp;&lt;nTableThreads&gt;3&lt;/nTableThreads&gt; </kbd>
<br>If your server has fewer resources, stick to "1" for both of those settings.
<br>The nThreads systems for EDDGridFromFiles and EDDTableFromFiles were significantly improved.
These changes led to a huge speed improvement (e.g., 2X speedup when nThreads is set to 2 or more)
for the most challenging requests (when a large number of files must be processed to gather the results).
Expand Down Expand Up @@ -6953,7 +6952,7 @@ <h2><a class="selfLink" id="contact" href="#contact" rel="bookmark">Contact</a><

<br>&nbsp;
<hr>
<p>ERDDAP, Version 2.20
<p>ERDDAP, Version 2.21
<br><a href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html">Disclaimers</a> |
<a href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html#privacyPolicy">Privacy Policy</a>

Expand Down
2 changes: 1 addition & 1 deletion download/grids.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1248,7 +1248,7 @@ <h3><a class="selfLink" id="RemoteReplicationOfDatasets_Solutions" href="#Remote

<br>&nbsp;
<hr>
<p>ERDDAP, Version 2.20
<p>ERDDAP, Version 2.21
<br><a rel="help" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html">Disclaimers</a> |
<a rel="help" href="https://coastwatch.pfeg.noaa.gov/erddap/legal.html#privacyPolicy">Privacy Policy</a>

Expand Down
Loading

0 comments on commit a041dda

Please sign in to comment.