From 22e8514ae94fd44af85bf13559c638e35d991068 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 20 Dec 2023 11:40:24 -0700 Subject: [PATCH 1/3] Cleanup useless print statements in tests --- .../test/java/ucar/nc2/dataset/TestDatasetUrl.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/cdm/core/src/test/java/ucar/nc2/dataset/TestDatasetUrl.java b/cdm/core/src/test/java/ucar/nc2/dataset/TestDatasetUrl.java index 209e7a8bdb..8b2f43a2f1 100644 --- a/cdm/core/src/test/java/ucar/nc2/dataset/TestDatasetUrl.java +++ b/cdm/core/src/test/java/ucar/nc2/dataset/TestDatasetUrl.java @@ -21,7 +21,6 @@ */ public class TestDatasetUrl { private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - static final boolean show = true; protected void protocheck(String path, String expected) { if (expected == null) @@ -32,11 +31,6 @@ protected void protocheck(String path, String expected) { StringBuffer buff = new StringBuffer(); protocols.forEach(p -> buff.append(p).append(":")); String result = buff.toString(); - boolean ok = expected.equals(result); - if (show || !ok) - System.out.printf(" %s <- %s%n", result, path); - if (!ok) - System.out.printf(" !!!EXPECTED '%s'%n", expected); assertThat(result).isEqualTo(expected); } @@ -76,11 +70,6 @@ public void testGetProtocols() { protected void testFind(String path, ServiceType expected) throws IOException { DatasetUrl result = DatasetUrl.findDatasetUrl(path); - boolean ok = (expected == null) ? result.serviceType == null : expected == result.serviceType; - if (show || !ok) - System.out.printf(" %s <- %s%n", result.serviceType, path); - if (!ok) - System.out.printf(" !!!EXPECTED '%s'%n", expected); assertThat(result.serviceType).isEqualTo(expected); } From 67f642d73999182ef93e8558db7e0891f6394dbb Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 20 Dec 2023 11:43:04 -0700 Subject: [PATCH 2/3] Add unit tests for creating DatasetUrls for ncml files --- .../test/java/ucar/nc2/dataset/TestDatasetUrl.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cdm/core/src/test/java/ucar/nc2/dataset/TestDatasetUrl.java b/cdm/core/src/test/java/ucar/nc2/dataset/TestDatasetUrl.java index 8b2f43a2f1..79aa15464b 100644 --- a/cdm/core/src/test/java/ucar/nc2/dataset/TestDatasetUrl.java +++ b/cdm/core/src/test/java/ucar/nc2/dataset/TestDatasetUrl.java @@ -112,5 +112,16 @@ public void testFindDatasetUrl() throws IOException { testFind( "dynamic:http://thredds.ucar.edu:8080/thredds/fmrc/NCEP/GFS/CONUS_95km/files/GFS_CONUS_95km_20070319_0600.grib1", null); + + testFind("file:/path/to/file.ncml", ServiceType.NCML); + testFind("/path/to/file.ncml", ServiceType.NCML); + testFind("file:/path/to/file.xml", ServiceType.NCML); + testFind("/path/to/file.xml", ServiceType.NCML); + + testFind("cdms3:thredds-test-data?testStandalone.ncml#delimiter=/", ServiceType.NCML); + testFind("cdms3:thredds-test-data?testStandalone.ncml", ServiceType.NCML); + testFind("cdms3:thredds-test-data?ncml/testStandalone.ncml#delimiter=/", ServiceType.NCML); + testFind("cdms3:thredds-test-data?ncml/testStandalone.ncml", ServiceType.NCML); + testFind("cdms3://profile_name@my.endpoint.edu/bucket-name?super/long/key.ncml#delimiter=/", ServiceType.NCML); } } From aa7a526648d5cc78a0f6e437636ca6d974609f82 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 22 Dec 2023 10:44:21 -0700 Subject: [PATCH 3/3] Make NcML reading work with S3 files --- .../main/java/thredds/filesystem/MFileOS.java | 2 +- .../java/ucar/nc2/dataset/DatasetUrl.java | 21 +++++++------ .../ucar/nc2/internal/ncml/NcmlReader.java | 16 ++++------ .../main/java/ucar/nc2/util/URLnaming.java | 31 ++++++++++++++++--- 4 files changed, 46 insertions(+), 24 deletions(-) diff --git a/cdm/core/src/main/java/thredds/filesystem/MFileOS.java b/cdm/core/src/main/java/thredds/filesystem/MFileOS.java index a5739c28b1..3bb3e7ff1a 100644 --- a/cdm/core/src/main/java/thredds/filesystem/MFileOS.java +++ b/cdm/core/src/main/java/thredds/filesystem/MFileOS.java @@ -53,7 +53,7 @@ public MFileOS(java.io.File file) { } public MFileOS(String filename) { - this.file = new File(filename); + this.file = new File(filename.replaceFirst("^file:", "")); this.lastModified = file.lastModified(); } diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/DatasetUrl.java b/cdm/core/src/main/java/ucar/nc2/dataset/DatasetUrl.java index 39e56ebd95..c88521cac8 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/DatasetUrl.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/DatasetUrl.java @@ -13,6 +13,8 @@ import com.google.common.collect.Multimap; import thredds.client.catalog.ServiceType; +import thredds.inventory.MFile; +import thredds.inventory.MFiles; import ucar.httpservices.HTTPFactory; import ucar.httpservices.HTTPMethod; import ucar.nc2.util.EscapeStrings; @@ -59,9 +61,11 @@ public static List getProtocols(String url) { StringBuilder buf = new StringBuilder(url); // If there are any leading protocols, then they must stop at the first '/'. int slashpos = buf.indexOf("/"); - // Check special case of file: with no slashes after file: + // Check special cases of file: or cdms3: with no slashes after: if (url.startsWith("file:") && "/\\".indexOf(url.charAt(5)) < 0) { allprotocols.add("file"); + } else if (url.startsWith("cdms3:") && "/\\".indexOf(url.charAt(6)) < 0) { + allprotocols.add("cdms3"); } else if (slashpos >= 0) { // Remove everything after the first slash buf.delete(slashpos + 1, buf.length()); @@ -128,7 +132,7 @@ public static DatasetUrl findDatasetUrl(String orgLocation) throws IOException { } pos = location.lastIndexOf('?'); String query = null; - if (pos >= 0) { + if (pos >= 0 && !leadProtocol.equals("cdms3")) { query = trueUrl.substring(pos + 1); trueUrl = trueUrl.substring(0, pos); } @@ -147,9 +151,9 @@ public static DatasetUrl findDatasetUrl(String orgLocation) throws IOException { // - we have file:// or file:; we need to see if // the extension can help, otherwise, start defaulting. // - we have a simple url: e.g. http://... ; contact the server - if (leadProtocol.equals("file")) { + if (leadProtocol.equals("file") || leadProtocol.equals("cdms3")) { serviceType = decodePathExtension(trueUrl); // look at the path extension - if (serviceType == null && checkIfNcml(new File(location))) { + if (serviceType == null && checkIfNcml(MFiles.create(location))) { serviceType = ServiceType.NCML; } } else { @@ -163,11 +167,10 @@ public static DatasetUrl findDatasetUrl(String orgLocation) throws IOException { } } } - if (serviceType == ServiceType.NCML) { // ?? // If lead protocol was null, then pretend it was a file // Note that technically, this should be 'file://' - trueUrl = (allProtocols.isEmpty() ? "file:" + trueUrl : location); + trueUrl = (allProtocols.isEmpty() ? "file:" + trueUrl : trueUrl); } // Add back the query and fragment (if any) @@ -511,12 +514,12 @@ private static boolean checkIfRemoteNcml(String location) throws IOException { return false; } - private static boolean checkIfNcml(File file) throws IOException { - if (!file.exists()) { + private static boolean checkIfNcml(MFile mFile) throws IOException { + if (!mFile.exists()) { return false; } - try (BufferedInputStream in = new BufferedInputStream(new FileInputStream(file), NUM_BYTES_TO_DETERMINE_NCML)) { + try (BufferedInputStream in = new BufferedInputStream(mFile.getInputStream(), NUM_BYTES_TO_DETERMINE_NCML)) { byte[] bytes = new byte[NUM_BYTES_TO_DETERMINE_NCML]; int bytesRead = in.read(bytes); diff --git a/cdm/core/src/main/java/ucar/nc2/internal/ncml/NcmlReader.java b/cdm/core/src/main/java/ucar/nc2/internal/ncml/NcmlReader.java index 104f7fa07a..9d316d9d02 100644 --- a/cdm/core/src/main/java/ucar/nc2/internal/ncml/NcmlReader.java +++ b/cdm/core/src/main/java/ucar/nc2/internal/ncml/NcmlReader.java @@ -18,13 +18,14 @@ import java.util.Optional; import java.util.Set; import java.util.StringTokenizer; -import java.net.URL; import javax.annotation.Nullable; import org.jdom2.Element; import org.jdom2.JDOMException; import org.jdom2.Namespace; import org.jdom2.input.SAXBuilder; import org.jdom2.output.XMLOutputter; +import thredds.inventory.MFile; +import thredds.inventory.MFiles; import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.nc2.Attribute; @@ -346,16 +347,11 @@ public static NetcdfDataset.Builder readNcml(Reader r, String ncmlLocation, Canc */ public static NetcdfDataset.Builder readNcml(String ncmlLocation, String referencedDatasetUri, CancelTask cancelTask) throws IOException { - URL url = new URL(ncmlLocation); + MFile mFile = MFiles.create(ncmlLocation); if (debugURL) { System.out.println(" NcmlReader open " + ncmlLocation); - System.out.println(" URL = " + url); - System.out.println(" external form = " + url.toExternalForm()); - System.out.println(" protocol = " + url.getProtocol()); - System.out.println(" host = " + url.getHost()); - System.out.println(" path = " + url.getPath()); - System.out.println(" file = " + url.getFile()); + System.out.println(" Path = " + mFile.getPath()); } org.jdom2.Document doc; @@ -363,9 +359,9 @@ public static NetcdfDataset.Builder readNcml(String ncmlLocation, String referen SAXBuilder builder = new SAXBuilder(); builder.setExpandEntities(false); if (debugURL) { - System.out.println(" NetcdfDataset URL = <" + url + ">"); + System.out.println(" NetcdfDataset path = <" + mFile.getPath() + ">"); } - doc = builder.build(url); + doc = builder.build(mFile.getInputStream()); } catch (JDOMException e) { throw new IOException(e.getMessage()); } diff --git a/cdm/core/src/main/java/ucar/nc2/util/URLnaming.java b/cdm/core/src/main/java/ucar/nc2/util/URLnaming.java index 60555acb1b..4d624f58d2 100644 --- a/cdm/core/src/main/java/ucar/nc2/util/URLnaming.java +++ b/cdm/core/src/main/java/ucar/nc2/util/URLnaming.java @@ -4,6 +4,9 @@ */ package ucar.nc2.util; +import java.io.IOException; +import thredds.inventory.MFile; +import thredds.inventory.MFiles; import ucar.unidata.util.StringUtil2; import java.io.File; import java.net.URI; @@ -86,6 +89,20 @@ public static String resolve(String baseUri, String relativeUri) { } } + if (baseUri.startsWith("cdms3:")) { + if (relativeUri.startsWith("cdms3:")) { + return relativeUri; + } else { + MFile absoluteMFile; + try { + absoluteMFile = MFiles.create(baseUri).getParent().getChild(relativeUri); + } catch (IOException e) { + return relativeUri; + } + return absoluteMFile == null ? relativeUri : absoluteMFile.getPath(); + } + } + // non-file URLs // relativeUri = canonicalizeRead(relativeUri); @@ -116,12 +133,18 @@ public static String resolveFile(String baseDir, String filepath) { if (baseDir.startsWith("file:")) baseDir = baseDir.substring(5); - File base = new File(baseDir); - if (!base.isDirectory()) - base = base.getParentFile(); + MFile base = MFiles.create(baseDir); + if (!base.isDirectory()) { + try { + base = base.getParent(); + } catch (IOException e) { + return filepath; + } + } if (base == null) return filepath; - return base.getAbsolutePath() + "/" + filepath; + MFile absoluteFile = base.getChild(filepath); + return absoluteFile == null ? filepath : absoluteFile.getPath(); } }