Skip to content

Commit

Permalink
v2.16
Browse files Browse the repository at this point in the history
  • Loading branch information
Bob Simons committed Dec 17, 2021
1 parent 617c2d5 commit 5e347db
Show file tree
Hide file tree
Showing 49 changed files with 736 additions and 505 deletions.
2 changes: 1 addition & 1 deletion WEB-INF/ArchiveADataset.bat
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ rem This is the Windows batch file to run ArchiveADataset.
rem See http://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html#Tools

rem You'll need to change java's path to make this work:
C:\programs\jdk8u292-b10\bin\java.exe -cp classes;../../../lib/servlet-api.jar;lib/* -Xms1500M -Xmx1500M gov.noaa.pfel.erddap.ArchiveADataset %*
C:\programs\jdk8u312-b07\bin\java.exe -cp classes;../../../lib/servlet-api.jar;lib/* -Xms1500M -Xmx1500M gov.noaa.pfel.erddap.ArchiveADataset %*
2 changes: 1 addition & 1 deletion WEB-INF/DasDds.bat
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ rem This is the Windows batch file to run DasDds.
rem See http://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html#Tools

# You'll need to change java's path to make this work:
C:\programs\jdk8u292-b10\bin\java.exe -cp classes;../../../lib/servlet-api.jar;lib/* -Xms1000M -Xmx1000M gov.noaa.pfel.erddap.DasDds %*
C:\programs\jdk8u312-b07\bin\java.exe -cp classes;../../../lib/servlet-api.jar;lib/* -Xms1000M -Xmx1000M gov.noaa.pfel.erddap.DasDds %*
2 changes: 1 addition & 1 deletion WEB-INF/FindDuplicateTime.bat
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ rem Parameters: directory fileNameRegex timeVarName
rem This is recursive.
rem Written by Bob Simons original 2021-01-20

C:\programs\jdk8u292-b10\bin\java.exe -cp classes;../../../lib/servlet-api.jar;lib/* -Xms1300M -Xmx1300M -verbose:gc gov.noaa.pfel.erddap.dataset.FindDuplicateTime %*
C:\programs\jdk8u312-b07\bin\java.exe -cp classes;../../../lib/servlet-api.jar;lib/* -Xms1300M -Xmx1300M -verbose:gc gov.noaa.pfel.erddap.dataset.FindDuplicateTime %*
2 changes: 1 addition & 1 deletion WEB-INF/GenerateDatasetsXml.bat
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@ rem This is the Windows batch file to run GenerateDatasetsXml.
rem See http://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html#Tools

rem You'll need to change java's path to make this work:
C:\programs\jdk8u292-b10\bin\java.exe -cp classes;../../../lib/servlet-api.jar;lib/* -Xms1000M -Xmx1000M gov.noaa.pfel.erddap.GenerateDatasetsXml %*
C:\programs\jdk8u312-b07\bin\java.exe -cp classes;../../../lib/servlet-api.jar;lib/* -Xms1000M -Xmx1000M gov.noaa.pfel.erddap.GenerateDatasetsXml %*
2 changes: 1 addition & 1 deletion WEB-INF/NetCheck.bat
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ rem Then run this from an MS-DOS Prompt (command line) window.
# Use "NetCheck -testMode" to just send emails to the NetCheck administrator.

set thisDir=%~dp0
java -cp classes;../../../lib/servlet-api.jar;lib/* -Xms1000M -Xmx1000M gov.noaa.pfel.coastwatch.netcheck.NetCheck NetCheck.xml %*
java -cp classes;../../../lib/servlet-api.jar;lib/* -Xms1400M -Xmx1400M gov.noaa.pfel.coastwatch.netcheck.NetCheck NetCheck.xml %*
2 changes: 1 addition & 1 deletion WEB-INF/NetCheck.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,4 @@
commandLine=$0
tDir=`echo $commandLine | dirname $0`

java -cp classes:../../../lib/servlet-api.jar:lib/* -Xms1000M -Xmx1000M gov.noaa.pfel.coastwatch.netcheck.NetCheck NetCheck.xml "$@"
java -cp classes:../../../lib/servlet-api.jar:lib/* -Xms1400M -Xmx1400M gov.noaa.pfel.coastwatch.netcheck.NetCheck NetCheck.xml "$@"
28 changes: 17 additions & 11 deletions WEB-INF/classes/com/cohort/array/StringArray.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,12 +294,16 @@ public StringArray(Map map) {
} */

/**
* This reads the text contents of the specified file using this computer's default charset.
*
* This reads the text contents of the specified file using File2.ISO_8859_1.
*/
public static StringArray fromFile(String fileName) throws Exception {

return fromFile(fileName, null);
public static StringArray fromFile88591(String fileName) throws Exception {
return fromFile(fileName, File2.ISO_8859_1);
}
/**
* This reads the text contents of the specified file using File2.UTF_8.
*/
public static StringArray fromFileUtf8(String fileName) throws Exception {
return fromFile(fileName, File2.UTF_8);
}


Expand Down Expand Up @@ -2508,11 +2512,12 @@ public int[] getNMinMaxIndex() {
*
* @param fileName1 a complete file name
* @param fileName2 a complete file name
* @param charset e.g., File2.ISO_8859_1 or File2.UTF_8.
* @throws Exception if files are different
*/
public static void diff(String fileName1, String fileName2) throws Exception {
StringArray sa1 = fromFile(fileName1);
StringArray sa2 = fromFile(fileName2);
public static void diff(String fileName1, String fileName2, String charset) throws Exception {
StringArray sa1 = fromFile(fileName1, charset);
StringArray sa2 = fromFile(fileName2, charset);
sa1.diff(sa2);
}

Expand All @@ -2523,15 +2528,16 @@ public static void diff(String fileName1, String fileName2) throws Exception {
*
* @param fileName1 a complete file name
* @param fileName2 a complete file name
* @param charset e.g., File2.ISO_8859_1 or File2.UTF_8.
* @throws Exception if files are different
*/
public static void repeatedDiff(String fileName1, String fileName2) throws Exception {
public static void repeatedDiff(String fileName1, String fileName2, String charset) throws Exception {
while (true) {
try {
String2.log("\nComparing " + fileName1 +
"\n and " + fileName2);
StringArray sa1 = fromFile(fileName1);
StringArray sa2 = fromFile(fileName2);
StringArray sa1 = fromFile(fileName1, charset);
StringArray sa2 = fromFile(fileName2, charset);
sa1.diff(sa2);
String2.log("!!! The files are the same!!!");
break;
Expand Down
20 changes: 10 additions & 10 deletions WEB-INF/classes/com/cohort/util/TestUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1887,10 +1887,10 @@ public static void testString2() throws Throwable {
//for (int i = 0; i < 10000000; i++)
// sb.append("a");
//String s9 = sb2.toString();
//File2.writeToFile(fileName, s9); //warm up
//File2.writeToFile(fileName, s9);
//File2.writeToFile88591(fileName, s9); //warm up
//File2.writeToFile88591(fileName, s9);
//long tTime = System.currentTimeMillis();
//File2.writeToFile(fileName, s9);
//File2.writeToFile88591(fileName, s9);
//String2.log("time=" + (System.currentTimeMillis() - tTime));
//sb2 = null;
//s9 = null;
Expand Down Expand Up @@ -2973,13 +2973,13 @@ public static void interactiveTestCalendar2() throws Exception {
"\nOracle Java versus AdoptOpenJDK may be slightly different for these.");
}

String2.log("getCurrentISODateTimeString = " + Calendar2.formatAsISODateTimeT(Calendar2.newGCalendarLocal()));
String2.log(" *** Check that HOUR accounts for daylight saving time!\n");
String2.log("current UTC time = " + Calendar2.formatAsISODateTimeT(Calendar2.newGCalendarZulu()));
String2.log(" *** Check that HOUR is local+7 in summer (DST), local+8 in winter (in California).");
String2.log(" See current UTC time at http://www.xav.com/time.cgi");
String2.pressEnterToContinue();

String2.pressEnterToContinue(
"getCurrentISODateTimeString = " + Calendar2.formatAsISODateTimeT(Calendar2.newGCalendarLocal()) +
" *** Check that HOUR accounts for daylight saving time!\n\n" +
"current UTC time = " + Calendar2.formatAsISODateTimeT(Calendar2.newGCalendarZulu()) + "\n" +
" *** Check that HOUR is local+7 in summer (DST), local+8 in winter (in California)\n" +
" or local+4 in summer (DST), local+5 in winter (on East Coast).\n" +
" See current UTC time at http://www.xav.com/time.cgi");
}

/**
Expand Down
6 changes: 3 additions & 3 deletions WEB-INF/classes/com/cohort/util/Units2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1796,7 +1796,7 @@ public static HashMap<String, String> getHashMapStringString(String fileName, St
*/
public static void checkUdunits2File(String fileName) throws Exception {
String2.log("\n*** Units2.checkUdUnits2File " + fileName);
StringArray sa = StringArray.fromFile(fileName);
StringArray sa = StringArray.fromFileUtf8(fileName);
StringArray names = new StringArray();
int n = sa.size();
int nDef = 0, nPlural = 0, nSingular = 0, nSymbol = 0;
Expand Down Expand Up @@ -1952,7 +1952,7 @@ public static StringArray getTestUdunits() throws Exception {
f == 1? "datasetsFEDCW.xml" :
"uniqueCFUnits.txt");

ArrayList<String> lines = File2.readLinesFromFile(fileName, File2.ISO_8859_1, 1); //nAttempts
ArrayList<String> lines = File2.readLinesFromFile(fileName, File2.UTF_8, 1); //nAttempts
int nLines = lines.size();
for (int i = 0; i < nLines; i++) {
String s = String2.extractCaptureGroup(lines.get(i), pattern, 1); //captureGroupNumber
Expand Down Expand Up @@ -2207,7 +2207,7 @@ else if (Calendar2.isStringTimeUnits(oUnits))
* This tests if the canonical units for each unique CF unit is unique.
*/
public static void testIfCFCanonicalUnitsUnique() throws Throwable {
StringArray sa = StringArray.fromFile(cfUnique, File2.UTF_8);
StringArray sa = StringArray.fromFileUtf8(cfUnique);
Attributes atts = new Attributes(); //use it as a hashmap: canon -> source
for (int i = 0; i < sa.size(); i++) {
String s = sa.get(i);
Expand Down
19 changes: 9 additions & 10 deletions WEB-INF/classes/gov/noaa/pfel/coastwatch/Projects.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3317,11 +3317,11 @@ public static void testHdf4() throws Exception {
String[] list = RegexFilenameFilter.recursiveFullNameList(
"c:/programs/_tomcat/webapps/cwexperimental/WEB-INF/classes/ucar4", ".*\\.java", false);
for (int i = 0; i < list.length; i++) {
String content[] = File2.readFromFile(list[i]);
String content[] = File2.readFromFileUtf8(list[i]);
if (content[0].length() == 0) {
String2.log("processing " + list[i]);
content[1] = String2.replaceAll(content[1], "ucar.", "ucar4.");
File2.writeToFile(list[i], content[1]);
File2.writeToFileUtf8(list[i], content[1]);
} else {
String2.log(content[0]);
}
Expand Down Expand Up @@ -6367,7 +6367,7 @@ public static void nodcPJJU(
for (int f = 0; f < fileNames.length; f++) {
try {
String2.log("\n#" + f + " " + fileNames[f]);
ArrayList<String> lines = File2.readLinesFromFile(inDir + fileNames[f], null, 2);
ArrayList<String> lines = File2.readLinesFromFile(inDir + fileNames[f], File2.ISO_8859_1, 2);

//BOTTLE,20030711WHPSIODMB
//#code : jjward hyd_to_exchange.pl
Expand Down Expand Up @@ -6984,10 +6984,9 @@ public static String makeNetcheckErddapTests(String erddapUrl) throws Throwable
*/
public static void fixKeywords(String fileName) throws Exception {
String2.log("fixKeywords " + fileName);
String charset = File2.ISO_8859_1;
String attKeywords = "<att name=\"keywords\">";
int attKeywordsLength = attKeywords.length();
StringArray lines = StringArray.fromFile(fileName, charset);
StringArray lines = StringArray.fromFileUtf8(fileName);
int linesSize = lines.size();
for (int i = 0; i < linesSize; i++) {
//start keywords?
Expand All @@ -7012,7 +7011,7 @@ public static void fixKeywords(String fileName) throws Exception {
}
}
}
lines.toFile(fileName);
lines.toFile(fileName, File2.UTF_8, null);
}

/**
Expand Down Expand Up @@ -7339,7 +7338,7 @@ public static void convertGlobecCsvToNc() throws Exception {
public static void splitRockfish() throws Exception {
String2.log("\n*** splitRockfish");
String fileDir = "c:/data/globec/";
StringArray sa = StringArray.fromFile(fileDir + "rockfish_view.csv");
StringArray sa = StringArray.fromFile88591(fileDir + "rockfish_view.csv");
int size = sa.size();

//this file has \N for missing values. change to ""
Expand All @@ -7352,7 +7351,7 @@ public static void splitRockfish() throws Exception {
while (size > 1) {
int stop = Math.min(100000, size - 1);
sa.subset(subset, 0, 1, stop); //stop is inclusive
subset.toFile(fileDir + "rockfish" + (chunk++) + ".csv");
subset.toFile(fileDir + "rockfish" + (chunk++) + ".csv", File2.ISO_8859_1, null);
sa.removeRange(1, stop + 1); //end is exclusive
size = sa.size();
}
Expand Down Expand Up @@ -9504,7 +9503,7 @@ public static void makePH2Ncml(String sstdn)

//String names[] = RegexFilenameFilter.list(dir, regex);
ArrayList<String> names = File2.readLinesFromFile(
"/u00/satellite/PH2/" + sstdn + "/names.txt", null, 1);
"/u00/satellite/PH2/" + sstdn + "/names.txt", File2.ISO_8859_1, 1);

//for each file
int nNames = names.size();
Expand Down Expand Up @@ -9559,7 +9558,7 @@ public static void makePH53Ncml()
for (int daynight = 0; daynight < 2; daynight++) {

ArrayList<String> names = File2.readLinesFromFile(
ncmlDir + dayNight[daynight] + "names" + year + ".txt", null, 1);
ncmlDir + dayNight[daynight] + "names" + year + ".txt", File2.ISO_8859_1, 1);
HashMap<String,String> hm = new HashMap();
//for each file
//19811101145206-NODC-L3C_GHRSST-SSTskin-AVHRR_Pathfinder-PFV5.2_NOAA07_G_1981305_day-v02.0-fv01.0.nc
Expand Down
21 changes: 11 additions & 10 deletions WEB-INF/classes/gov/noaa/pfel/coastwatch/TestAll.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ public static void main(String args[]) throws Throwable {
} /* */

// String tFileName = String2.unitTestBigDataDir + "nccf/wod/wod_xbt_2005.nc";
// File2.writeToFile(tFileName + ".ncdump.txt", NcHelper.ncdump(tFileName, "-h"));
// File2.writeToFileUtf8(tFileName + ".ncdump.txt", NcHelper.ncdump(tFileName, "-h"));

// String2.log(EDDGrid.findTimeGaps("https://coastwatch.pfeg.noaa.gov/erddap/griddap/nceiPH53sstn1day"));

Expand Down Expand Up @@ -329,7 +329,7 @@ public static void main(String args[]) throws Throwable {
*/

// s = Projects.makeAwsS3FilesDatasets(".*"); //all: .* tests: nrel-pds-wtk\\.yaml, silo\\.yaml (has ?Name: but ? doesn't show in EditPlus)
// File2.writeToFile(EDStatic.bigParentDirectory + "logs/awsS3Files.txt", s);
// File2.writeToFileUtf8(EDStatic.bigParentDirectory + "logs/awsS3Files.txt", s);
// String2.log(EDD.testDasDds("radar_vola"));

/*
Expand Down Expand Up @@ -443,7 +443,7 @@ public static void main(String args[]) throws Throwable {

/* //tallyXml
String tfn = EDStatic.fullLogsDirectory + "tallyLterSbsStorageUnitsMV.log";
File2.writeToFile(tfn,
File2.writeToFileUtf8(tfn,
FileVisitorDNLS.tallyXml(
"/u00/data/points/lterSbc/", "knb-lter-sbc\\.\\d+", false,
new String[]{
Expand Down Expand Up @@ -474,11 +474,11 @@ public static void main(String args[]) throws Throwable {
// Table.debugMode = true; DasDds.main(new String[]{"NTL_DEIMS_5672_t1", "-verbose"});

// make flag files for all knb datasets
// ArrayList<String> tsa = File2.readLinesFromFile("/downloads/allKnb.txt", "", 1);
// ArrayList<String> tsa = File2.readLinesFromFile("/downloads/allKnb.txt", File2.ISO_8859_1, 1);
// int nTsa = tsa.size();
// String2.log("allKnb n=" + nTsa);
// for (int tsai = 0; tsai < nTsa; tsai++)
// File2.writeToFile("/flag/" + tsa.get(tsai), "flag");
// File2.writeToFileUtf8("/flag/" + tsa.get(tsai), "flag");

// EDDTableFromHttpGet.testStatic();
// String2.log(EDDTableFromHyraxFiles.generateDatasetsXml(
Expand Down Expand Up @@ -551,6 +551,7 @@ public static void main(String args[]) throws Throwable {
// was from GTSPP dir: /nodc/data/gtspp/bestcopy/netcdf
// was /pub/data.nodc/gtspp/bestcopy/netcdf
// to my local: c:/data/gtspp/bestNcZip
// !!! Be sure do pre-delete older files, so you don't download as filename(1) .
// !!! Note that older files are reprocessed sometimes.
// !!! So sort by lastModified time to check if "older" files have a recent last-modified-time.

Expand All @@ -566,7 +567,7 @@ public static void main(String args[]) throws Throwable {
// !!! CLOSE all other windows, even EditPlus.
// !!! EMPTY Recycle Bin
// !!! CHANGE "Run TestAll" MEMORY SETTING to 7GB
// EDDTableFromNcFiles.bobConsolidateGtsppTgz(2021, 8, 2021, 10, false); //first/last year(1985..)/month(1..), testMode 1985,02 is first time
// EDDTableFromNcFiles.bobConsolidateGtsppTgz(2021, 9, 2021, 11, false); //first/last year(1985..)/month(1..), testMode 1985,02 is first time
// log file is c:/data/gtspp/logYYYYMMDD.txt
// 2b) Email the "good" but "impossible" stations to Tim Boyer <[email protected]>,
// and "Christopher Paver - NOAA Federal ([email protected])" <[email protected]>
Expand Down Expand Up @@ -599,7 +600,7 @@ public static void main(String args[]) throws Throwable {
// It takes ~20 seconds per month processed.
// It uses a local version of the dataset, not the one in localhost erddap.
// !!! CHANGE TestAll MEMORY SETTING to 7GB //2016-10 is huge//
// EDDTableFromNcFiles.bobCreateGtsppNcCFFiles(2021, 8, 2021, 10); //e.g., first/last year(1985..)/month(1..)
// EDDTableFromNcFiles.bobCreateGtsppNcCFFiles(2021, 9, 2021, 11); //e.g., first/last year(1985..)/month(1..)
// String2.log(NcHelper.ncdump("/u00/data/points/gtsppNcCF/201406a.nc", "-h"));
// 8) Run: (should fail at current calendar month)
// EDDTableFromNcFiles.testGtsppabFilesExist(1990, 2021);
Expand Down Expand Up @@ -784,7 +785,7 @@ public static void main(String args[]) throws Throwable {
// SSR.testForBrokenLinks("http://localhost:8080/cwexperimental/images/erddapTalk/erdData.html");
// SSR.testForBrokenLinks("g:/NOAA-Navy-SanctSound_CI01_01_BB_1h.xml");
//
// StringArray.repeatedDiff("c:/downloads/f1.txt", "c:/downloads/f2.txt");
// StringArray.repeatedDiff("c:/downloads/f1.txt", "c:/downloads/f2.txt", File2.UTF_8);
// XML.prettyXml("c:/programs/mapserver/WVBoreholeResponse.xml",
// "c:/programs/mapserver/WVBoreholeResponsePretty.xml");

Expand Down Expand Up @@ -1390,9 +1391,9 @@ public static void main(String args[]) throws Throwable {

if (errorSB != null && errorSB.length() > 0) {
String fileName = EDStatic.fullLogsDirectory + "/TestAllErrorSB.txt";
String2.log(File2.writeToFile(fileName,
String2.log(File2.writeToFileUtf8(fileName,
"errorSB from TestAll which finished at " + Calendar2.getCurrentISODateTimeStringLocalTZ() + "\n" +
errorSB.toString(), File2.UTF_8));
errorSB.toString()));
SSR.displayInBrowser("file://" + fileName);
}

Expand Down
Loading

0 comments on commit 5e347db

Please sign in to comment.