Skip to content

Commit

Permalink
Load Resources with Guava API (#200)
Browse files Browse the repository at this point in the history 
* update gitignore

* gitignore build folder

* merge gitignore

* add override for classpath

* move resources to main source set

* begin fixing some resource and WEB-INF references

* changing more references to resources

* fix all references except for EDStatic

* fix some of the tests

* bring back WEB-INF lookup for EDStatic

* remove placeholder methods

* add resources to build script

* remove unused imports, fix test initialization, add back input stream decompression

* add new resources path to pom.xml, revert to static init via File2

* all tests should be passing now

* clean up stray import statements
  • Loading branch information
@SethChampagneNRL
SethChampagneNRL authored Sep 18, 2024
1 parent fdc67c9 commit ea5c5c5
Show file tree
Hide file tree
Showing 160 changed files with 507 additions and 325 deletions.
4 changes: 4 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ WEB-INF/docs
WEB-INF/lib/
WEB-INF/ref/
WEB-INF/temp/
WEB-INF/build/
images/erddapTalk
images/gadgets
images/post
Expand All @@ -15,6 +16,9 @@ Thumbs.db
.settings
.classpath
.project
/build/
/out/
/.gradle/
src/test/resources/*
!src/test/resources/datasets/
.idea
Expand Down
53 changes: 44 additions & 9 deletions WEB-INF/classes/com/cohort/array/StringArray.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,26 @@
*/
package com.cohort.array;

import com.cohort.util.*;
import com.cohort.util.File2;
import com.cohort.util.Math2;
import com.cohort.util.MustBe;
import com.cohort.util.SimpleException;
import com.cohort.util.String2;
import com.cohort.util.StringHolder;
import com.cohort.util.StringHolderComparator;
import com.cohort.util.StringHolderComparatorIgnoreCase;
import ucar.ma2.StructureData;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.math.BigInteger;
import java.net.URL;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
Expand All @@ -23,7 +34,6 @@
import java.util.Iterator;
import java.util.Set;
import java.util.regex.Pattern;
import ucar.ma2.StructureData;

/**
* StringArray is a thin shell over a String[] with methods like ArrayList's methods; it extends
Expand Down Expand Up @@ -287,6 +297,16 @@ public static StringArray fromFileUtf8(final String fileName) throws Exception {
return fromFile(fileName, File2.UTF_8);
}

/** This reads the text contents of the specified file using File2.ISO_8859_1. */
public static StringArray fromFile88591(final URL resourceFile) throws Exception {
return fromFile(resourceFile, File2.ISO_8859_1);
}

/** This reads the text contents of the specified file using File2.UTF_8. */
public static StringArray fromFileUtf8(final URL resourceFile) throws Exception {
return fromFile(resourceFile, File2.UTF_8);
}

/**
* This reads the text contents of the specified file and makes a StringArray with an item from
* each line of the file (not trim'd).
Expand All @@ -300,18 +320,33 @@ public static StringArray fromFile(final String fileName, final String charset)
Math2.ensureMemoryAvailable(
File2.length(fileName), "StringArray.fromFile"); // canonical may lessen memory requirement
final StringArray sa = new StringArray();
final BufferedReader bufferedReader =
File2.getDecompressedBufferedFileReader(fileName, charset);
try {
try (final BufferedReader bufferedReader = File2.getDecompressedBufferedFileReader(fileName, charset);) {
String s = bufferedReader.readLine();
while (s != null) { // null = end-of-file
sa.addNotCanonical(s);
s = bufferedReader.readLine();
}
} finally {
try {
bufferedReader.close();
} catch (Exception e) {
}
return sa;
}

/**
* This reads the text contents of the specified file and makes a StringArray with an item from
* each line of the file (not trim'd).
*
* @param charset e.g., ISO-8859-1 (the default); or "" or null for the default
* @return StringArray with not canonical strings (on the assumption that lines of all file are
* usually all different).
* @throws Exception if trouble (e.g., file not found)
*/
public static StringArray fromFile(final URL resourceFile, final String charset) throws Exception {
final StringArray sa = new StringArray();
try (InputStreamReader reader = new InputStreamReader(resourceFile.openStream(), charset);
BufferedReader bufferedReader = new BufferedReader(reader)){
String s = bufferedReader.readLine();
while (s != null) { // null = end-of-file
sa.addNotCanonical(s);
s = bufferedReader.readLine();
}
}
return sa;
Expand Down
192 changes: 135 additions & 57 deletions WEB-INF/classes/com/cohort/util/File2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLDecoder;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
Expand All @@ -29,6 +32,9 @@
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

import com.google.common.io.Resources;
import gov.noaa.pfel.erddap.util.EDStatic;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
Expand Down Expand Up @@ -57,8 +63,7 @@ public class File2 {
public static final Charset UTF_8_CHARSET = StandardCharsets.UTF_8;
public static final String UNABLE_TO_DELETE = "unable to delete"; // so consistent in log.txt

private static String classPath; // lazy creation by getClassPath
private static String webInfParentDirectory; // lazy creation by webInfParentDirectory()
private static String webInfParentDirectory;

// define the file types for the purpose of assigning icon in Table.directoryListing
// compressed and image ext from wikipedia
Expand Down Expand Up @@ -295,37 +300,25 @@ public class File2 {
private static ConcurrentHashMap<String, S3Client> s3ClientMap =
new ConcurrentHashMap<String, S3Client>();

/**
* This returns the directory that is the classpath for the source code files (with forward
* slashes and a trailing slash, e.g.,
* c:/programs/_tomcat/webapps/cwexperimental/WEB-INF/classes/.
*
* @return directory that is the classpath for the source code files (with / separator and / at
* the end)
* @throws RuntimeException if trouble
*/
public static String getClassPath() {
if (classPath == null) {
String find = "/com/cohort/util/String2.class";
// use this.getClass(), not ClassLoader.getSystemResource (which fails in Tomcat)
classPath = String2.class.getResource(find).getFile();
classPath = String2.replaceAll(classPath, '\\', '/');
int po = classPath.indexOf(find);
classPath = classPath.substring(0, po + 1);

// on windows, remove the troublesome leading "/"
if (String2.OSIsWindows
&& classPath.length() > 2
&& classPath.charAt(0) == '/'
&& classPath.charAt(2) == ':') classPath = classPath.substring(1);

// classPath is a URL! so spaces are encoded as %20 on Windows!
// UTF-8: see https://en.wikipedia.org/wiki/Percent-encoding#Current_standard
try {
classPath = URLDecoder.decode(classPath, UTF_8);
} catch (Throwable t) {
String2.log(MustBe.throwableToString(t));
}
String find = "/com/cohort/util/String2.class";
// use this.getClass(), not ClassLoader.getSystemResource (which fails in Tomcat)
String classPath = String2.class.getResource(find).getFile();
int po = classPath.indexOf(find);
classPath = classPath.substring(0, po + 1);

// on windows, remove the troublesome leading "/"
if (String2.OSIsWindows
&& classPath.length() > 2
&& classPath.charAt(0) == '/'
&& classPath.charAt(2) == ':') classPath = classPath.substring(1);

// classPath is a URL! so spaces are encoded as %20 on Windows!
// UTF-8: see https://en.wikipedia.org/wiki/Percent-encoding#Current_standard
try {
classPath = URLDecoder.decode(classPath, StandardCharsets.UTF_8);
} catch (Throwable t) {
String2.log(MustBe.throwableToString(t));
}

return classPath;
Expand All @@ -341,31 +334,47 @@ public static String getClassPath() {
* trouble
* @throws RuntimeException if trouble
*/
public static String webInfParentDirectory() {
if (webInfParentDirectory == null) {
String classPath = getClassPath(); // with / separator and / at the end
int po = classPath.indexOf("/WEB-INF/");
if (po < 0)
throw new RuntimeException(
String2.ERROR + ": '/WEB-INF/' not found in classPath=" + classPath);
webInfParentDirectory = classPath.substring(0, po + 1);
private static String lookupWebInfParentDirectory() {
String classPath = getClassPath();
int po = classPath.indexOf("/WEB-INF/");
if (po < 0)
throw new RuntimeException(
String2.ERROR + ": '/WEB-INF/' not found in classPath=" + classPath);
classPath = classPath.substring(0, po + 1);
Path path;
if (classPath.startsWith("file:/")) {
path = Paths.get(URI.create(classPath));
} else {
path = Paths.get(classPath);
}
return path.toAbsolutePath().toString().replace("\\", "/") + "/";
}

public static String getWebInfParentDirectory() {
if (webInfParentDirectory == null) {
webInfParentDirectory = lookupWebInfParentDirectory();
}
return webInfParentDirectory;
}

public static void setWebInfParentDirectory(String webInfParentDir) {
webInfParentDirectory = webInfParentDir.replace("\\", "/");
}

/**
* This sets the temp directory instead of looking for one based on where the WEB-INF directory
* is.
* Access a classpath resource via a filesystem path.
* NOTE: this will not work unless resource is exploded.
*
* <p>THIS IS ONLY INTENDED FOR USE DURING TESTS.
* @param resourcePath Classpath of resource.
* @return Filesystem path.
* @throws URISyntaxException Could not create URI.
*/
public static void setWebInfParentDirectory() {
webInfParentDirectory = System.getProperty("user.dir") + "/";
}

public static void setWebInfParentDirectory(String dir) {
webInfParentDirectory = dir;
public static String accessResourceFile(String resourcePath) throws URISyntaxException {
if (resourcePath.startsWith("file:/")) {
return Paths.get(new URI(resourcePath)).toString();
} else {
return Paths.get(resourcePath).toString();
}
}

/**
Expand Down Expand Up @@ -1209,12 +1218,10 @@ public static BufferedInputStream getBufferedInputStream(
* @return a decompressed, buffered InputStream from a file.
* @throws Exception if trouble
*/
public static InputStream getDecompressedBufferedInputStream(String fullFileName)
throws Exception {
public static InputStream getDecompressedBufferedInputStream(String fullFileName, InputStream is)
throws Exception {
String ext = getExtension(fullFileName); // if e.g., .tar.gz, this returns .gz

InputStream is = getBufferedInputStream(fullFileName); // starting point before decompression

// !!!!! IF CHANGE SUPPORTED COMPRESSION TYPES, CHANGE isDecompressible ABOVE !!!

// handle .Z (capital Z) specially first
Expand All @@ -1231,8 +1238,8 @@ public static InputStream getDecompressedBufferedInputStream(String fullFileName
if (ext.indexOf('z') < 0) return is;

if (ext.equals(".tgz")
|| fullFileName.endsWith(".tar.gz")
|| fullFileName.endsWith(".tar.gzip")) {
|| fullFileName.endsWith(".tar.gz")
|| fullFileName.endsWith(".tar.gzip")) {
// modified from
// https://stackoverflow.com/questions/7128171/how-to-compress-decompress-tar-gz-files-in-java
GzipCompressorInputStream gzipIn = null;
Expand All @@ -1244,7 +1251,7 @@ public static InputStream getDecompressedBufferedInputStream(String fullFileName
while (entry != null && entry.isDirectory()) entry = tarIn.getNextTarEntry();
if (entry == null)
throw new IOException(
String2.ERROR + " while reading " + fullFileName + ": no file found in archive.");
String2.ERROR + " while reading " + fullFileName + ": no file found in archive.");
is = tarIn;
} catch (Exception e) {
if (tarIn != null) tarIn.close();
Expand All @@ -1269,7 +1276,7 @@ public static InputStream getDecompressedBufferedInputStream(String fullFileName
while (entry != null && entry.isDirectory()) entry = zis.getNextEntry();
if (entry == null)
throw new IOException(
String2.ERROR + " while reading " + fullFileName + ": no file found in archive.");
String2.ERROR + " while reading " + fullFileName + ": no file found in archive.");
is = zis;
} catch (Exception e) {
if (zis != null) zis.close();
Expand All @@ -1292,6 +1299,35 @@ public static InputStream getDecompressedBufferedInputStream(String fullFileName
return is;
}

/**
* This gets a decompressed, buffered InputStream from a file or S3 URL. If the file is
* compressed, it is assumed to be the only file (entry) in the archive.
*
* @param fullFileName The full file name. If it ends in .tgz, .tar.gz, .tar.gzip, .gz, .gzip,
* .zip, or .bz2, this returns a decompressed, buffered InputStream.
* @return a decompressed, buffered InputStream from a file.
* @throws Exception if trouble
*/
public static InputStream getDecompressedBufferedInputStream(String fullFileName)
throws Exception {
InputStream is = getBufferedInputStream(fullFileName); // starting point before decompression
return getDecompressedBufferedInputStream(fullFileName, is);
}

/**
* This gets a decompressed, buffered InputStream from a file or S3 URL. If the file is
* compressed, it is assumed to be the only file (entry) in the archive.
*
* @param resourceFile The full file name. If it ends in .tgz, .tar.gz, .tar.gzip, .gz, .gzip,
* .zip, or .bz2, this returns a decompressed, buffered InputStream.
* @return a decompressed, buffered InputStream from a file.
* @throws Exception if trouble
*/
public static InputStream getDecompressedBufferedInputStream(URL resourceFile)
throws Exception {
return getDecompressedBufferedInputStream(resourceFile.getFile(), resourceFile.openStream());
}

public static void decompressAllFiles(String sourceFullName, String destDir) throws IOException {
String ext = getExtension(sourceFullName); // if e.g., .tar.gz, this returns .gz
// !!!!! IF CHANGE SUPPORTED COMPRESSION TYPES, CHANGE isDecompressible ABOVE
Expand Down Expand Up @@ -1726,6 +1762,48 @@ public static ArrayList<String> readLinesFromFile(String fileName, String charse
if (bufferedReader != null) bufferedReader.close();
}
}
/**
* This is like the other readFromFile, but returns ArrayList of Strings and throws Exception is
* trouble. The strings in the ArrayList are not canonical! So this is useful for reading,
* processing, and throwing away.
*
* <p>This method is generally appropriate for small and medium-sized files. For very large files
* or files that need additional processing, it may be more efficient to write a custom method to
* read the file line-by-line, processing as it goes.
*
* @param resourceFile URL of the file to be read
* @param charset e.g., ISO-8859-1, UTF-8, or "" or null for the default (ISO-8859-1)
* @param maxAttempt e.g. 3 (the tries are 1 second apart)
* @return ArrayList with the lines from the file
* @throws Exception if trouble
*/
public static ArrayList<String> readLinesFromFile(URL resourceFile, String charset, int maxAttempt)
throws Exception {

long time = System.currentTimeMillis();
BufferedReader bufferedReader = null;
try {
for (int i = 0; i < maxAttempt; i++) {
try {
InputStream is = getDecompressedBufferedInputStream(resourceFile);
bufferedReader = new BufferedReader(new InputStreamReader(is, charset));
break; // success
} catch (RuntimeException e) {
if (i == maxAttempt - 1) throw e;
Math2.sleep(100);
}
}
ArrayList<String> al = new ArrayList();
String s = bufferedReader.readLine();
while (s != null) { // null = end-of-file
al.add(s);
s = bufferedReader.readLine();
}
return al;
} finally {
if (bufferedReader != null) bufferedReader.close();
}
}

/*
Here is a skeleton for more direct control of reading text from a file:
Expand Down
Loading

0 comments on commit ea5c5c5

Please sign in to comment.