Skip to content

Commit

Permalink
Merge branch 'master' of https://github.com/opencadc/tap into polygon…
Browse files Browse the repository at this point in the history 
…-vertex-order-fix
  • Loading branch information
@at88mph
at88mph committed Feb 21, 2024
2 parents 03f57cb + 8bc296d commit 63cffbe
Show file tree
Hide file tree
Showing 77 changed files with 8,074 additions and 132 deletions.
17 changes: 9 additions & 8 deletions .github/workflows/gradle.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ jobs:
with:
java-version: 1.8

- name: build and test cadc-jsqlparser-compat
run: cd cadc-jsqlparser-compat && ../gradlew --info clean build javadoc install

- name: build and test cadc-tap
run: cd cadc-tap && ../gradlew --info clean build javadoc install

Expand All @@ -25,18 +28,16 @@ jobs:
- name: build and test cadc-tap-server
run: cd cadc-tap-server && ../gradlew --info clean build javadoc install

- name: build and test cadc-tap-server-pg
- name: build and test cadc-adql
run: cd cadc-adql && ../gradlew --info clean build javadoc install

- name: build and test cadc-tap-server-pg
run: cd cadc-tap-server-pg && ../gradlew --info clean build javadoc install

- name: build and test cadc-tap-server-oracle
run: cd cadc-adql && ../gradlew --info clean build javadoc install

- name: build and test cadc-jsqlparser-compat
run: cd cadc-jsqlparser-compat && ../gradlew --info clean build javadoc install

- name: build and test cadc-adql
run: cd cadc-adql && ../gradlew --info clean build javadoc install



- name: build and test cadc-test-tap
run: cd cadc-test-tap && ../gradlew --info clean build javadoc install

Expand Down
5 changes: 5 additions & 0 deletions cadc-adql/src/test/java/ca/nrc/cadc/tap/permissions/TapSchemaReadAccessConverterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,11 @@ protected void init() {

static class TestIdentityManager implements IdentityManager {

@Override
public Set<URI> getSecurityMethods() {
throw new UnsupportedOperationException();
}

@Override
public Subject validate(Subject subject) throws NotAuthenticatedException {
throw new UnsupportedOperationException();
Expand Down
2 changes: 1 addition & 1 deletion cadc-tap-server/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ sourceCompatibility = 1.8

group = 'org.opencadc'

version = '1.1.21'
version = '1.1.23'

description = 'OpenCADC TAP-1.1 tap server library'
def git_url = 'https://github.com/opencadc/tap'
Expand Down
170 changes: 118 additions & 52 deletions cadc-tap-server/src/main/java/ca/nrc/cadc/tap/DefaultTableWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,14 @@
import ca.nrc.cadc.uws.Job;
import ca.nrc.cadc.uws.ParameterUtil;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
import java.net.URI;
import java.net.URISyntaxException;
Expand Down Expand Up @@ -397,71 +401,133 @@ public void write(ResultSet rs, Writer out, Long maxrec) throws IOException
this.rowcount = tableData.getRowCount();
}

private void addMetaResources(VOTableDocument votableDocument, List<String> serviceIDs)
throws IOException
{
for (String serviceID : serviceIDs)
{
String filename = serviceID + ".xml";
InputStream is = DefaultTableWriter.class.getClassLoader().getResourceAsStream(filename);
if (is == null)
{
//throw new MissingResourceException(
// "Resource not found: " + serviceID + ".xml", DefaultTableWriter.class.getName(), filename);
log.debug("failed to find service resource " + filename + " to go with XML ID " + serviceID);
}
else
{
VOTableReader reader = new VOTableReader();
VOTableDocument serviceDocument = reader.read(is);
VOTableResource metaResource = serviceDocument.getResourceByType("meta");
votableDocument.getResources().add(metaResource);

// set the access URL from resourceIdentifier if possible
RegistryClient regClient = new RegistryClient();

try
{
// HACK: need to allow an ObsCore.access_url formatter to access this info
public static URL getAccessURL(String columnID, URI reqStandardID) throws IOException {
VOTableDocument serviceDocument = getDoc(columnID);
if (serviceDocument == null) {
return null;
}

String filename = columnID + ".xml"; // for error reporting
// find specified endpoint
for (VOTableResource metaResource : serviceDocument.getResources()) {
if ("meta".equals(metaResource.getType())) {
try {
URL accessURL = null;
URI resourceIdentifier = null;
URI standardID = null;
Iterator<VOTableParam> i = metaResource.getParams().iterator();
while ( i.hasNext() )
{
while (i.hasNext()) {
VOTableParam vp = i.next();
if (vp.getName().equals("resourceIdentifier"))
{
if (vp.getName().equals("accessURL")) {
accessURL = new URL(vp.getValue());
} else if (vp.getName().equals("resourceIdentifier")) {
resourceIdentifier = new URI(vp.getValue());
}
else if (vp.getName().equals("standardID"))
{
} else if (vp.getName().equals("standardID")) {
standardID = new URI(vp.getValue());
}
}
if (resourceIdentifier != null)
{
Subject s = AuthenticationUtil.getCurrentSubject();
AuthMethod cur = AuthenticationUtil.getAuthMethod(s);
if (cur == null)
{
cur = AuthMethod.ANON;
log.debug("getAccessURL: " + reqStandardID + " vs " + standardID);
if (reqStandardID.equals(standardID)) {
if (accessURL == null && resourceIdentifier != null && standardID != null) {
// try to augment resource with accessURL
Subject s = AuthenticationUtil.getCurrentSubject();
AuthMethod cur = AuthenticationUtil.getAuthMethod(s);
if (cur == null) {
cur = AuthMethod.ANON;
}
RegistryClient regClient = new RegistryClient();
log.debug("resourceIdentifier=" + resourceIdentifier + ", standardID=" + standardID + ", authMethod=" + cur);
accessURL = regClient.getServiceURL(resourceIdentifier, standardID, cur);
}
log.debug("resourceIdentifier=" + resourceIdentifier + ", standardID=" + standardID + ", authMethod=" + cur);
URL accessURL = regClient.getServiceURL(resourceIdentifier, standardID, cur);
if (accessURL != null) {
String surl = accessURL.toExternalForm();
String arraysize = Integer.toString(surl.length()); // fixed length since we know it
VOTableParam accessParam = new VOTableParam("accessURL", "char", arraysize, surl);
metaResource.getParams().add(accessParam);
} else {
// log the error but continue anyway
log.error("failed to find accessURL: resourceIdentifier=" + resourceIdentifier
+ ", standardID=" + standardID + ", authMethod=" + cur);
return accessURL;
}
}
} catch (URISyntaxException e) {
throw new RuntimeException("CONFIG: URI in " + filename + " is invalid", e);
}
catch (URISyntaxException e)
{
throw new RuntimeException("resourceIdentifier in " + filename + " is invalid", e);
}
}

return null;
}

private static VOTableDocument getDoc(String sid) throws IOException {
File configDir = new File(System.getProperty("user.home") + "/config");
String filename = sid + ".xml";
File tmpl = new File(configDir, filename);
Reader rdr = null;
if (tmpl.exists()) {
rdr = new FileReader(tmpl);
} else {
// backwards compat: classpath
InputStream is = DefaultTableWriter.class.getClassLoader().getResourceAsStream(filename);
if (is != null) {
rdr = new InputStreamReader(is);
}
}
if (rdr == null) {
log.debug("failed to find config resource " + filename + " to go with XML ID " + sid);
return null;
}

VOTableReader reader = new VOTableReader();
VOTableDocument serviceDocument = reader.read(rdr);
return serviceDocument;
}

private void addMetaResources(VOTableDocument votableDocument, List<String> serviceIDs)
throws IOException {
RegistryClient regClient = new RegistryClient();
for (String serviceID : serviceIDs) {
VOTableDocument serviceDocument = getDoc(serviceID);
String filename = serviceID + ".xml";
if (serviceDocument == null) {
return;
}

for (VOTableResource metaResource : serviceDocument.getResources()) {
if ("meta".equals(metaResource.getType())) {
votableDocument.getResources().add(metaResource);
try {
URL accessURL = null;
URI resourceIdentifier = null;
URI standardID = null;
Iterator<VOTableParam> i = metaResource.getParams().iterator();
while (i.hasNext()) {
VOTableParam vp = i.next();
if (vp.getName().equals("accessURL")) {
accessURL = new URL(vp.getValue());
} else if (vp.getName().equals("resourceIdentifier")) {
resourceIdentifier = new URI(vp.getValue());
} else if (vp.getName().equals("standardID")) {
standardID = new URI(vp.getValue());
}
}
if (accessURL == null && resourceIdentifier != null && standardID != null) {
// try to augment resource with accessURL
Subject s = AuthenticationUtil.getCurrentSubject();
AuthMethod cur = AuthenticationUtil.getAuthMethod(s);
if (cur == null) {
cur = AuthMethod.ANON;
}
log.debug("resourceIdentifier=" + resourceIdentifier + ", standardID=" + standardID + ", authMethod=" + cur);
accessURL = regClient.getServiceURL(resourceIdentifier, standardID, cur);
if (accessURL != null) {
String surl = accessURL.toExternalForm();
String arraysize = Integer.toString(surl.length()); // fixed length since we know it
VOTableParam accessParam = new VOTableParam("accessURL", "char", arraysize, surl);
metaResource.getParams().add(accessParam);
} else {
// log the error but continue anyway
log.error("failed to find accessURL: resourceIdentifier=" + resourceIdentifier
+ ", standardID=" + standardID + ", authMethod=" + cur);
}
}
} catch (URISyntaxException e) {
throw new RuntimeException("resourceIdentifier in " + filename + " is invalid", e);
}
}
}
}
Expand Down
42 changes: 27 additions & 15 deletions cadc-tap-tmp/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,47 +2,59 @@

Simple library to provide plugins that implement both the ResultStore (TAP-async
result storage) and UWSInlineContentHandler (inline TAP_UPLOAD support) interfaces.
Two interfaces are provided: `org.opencadc.tap.tmp.TempStorageManager` uses a configurable
Two implementations are provided: `org.opencadc.tap.tmp.TempStorageManager` uses a configurable
local directory in the filesystem, `org.opencadc.tap.tmp.HttpStorageManager` uses an
external HTTP service to store and deliver files.

## Applying cadc-tap-tmp
In addition, a third implementation `org.opencadc.tap.tmp.DelegatingStorageManager` can be
configured in services; this implementation can be configured to load and use one of the
other two classes above.

### Configuration
A configuration file `cadc-tap-tmp.properties` needs to be accessible in the `System.getProperty("user.home")/config` folder, or in a separate folder specified by a System property called `ca.nrc.cadc.util.PropertiesReader.dir`.
## configuration

### cadc-tap-tmp.properties
This configuration file needs to be accessible in the `System.getProperty("user.home")/config` folder.

For the local filesystem using `TempStorageManager`:
```properties
org.opencadc.tap.tmp.TempStorageManager.baseURL = {base URL for result files}
# specify implementation in case the DelegatingStorageManager is used
org.opencadc.tap.tmp.StorageManager = org.opencadc.tap.tmp.TempStorageManager

# TempStorageManager config
org.opencadc.tap.tmp.TempStorageManager.baseURL = {base URL for the tmp files}
org.opencadc.tap.tmp.TempStorageManager.baseStorageDir = {local directory for tmp files}
```
For the TempStorageManager, an additional servlet must be deployed in the TAP
service to [Enable Retrieval](#enable-retrieval)
service to [Enable Retrieval](#enable-retrieval) and the _baseURL_ must include
the path to that servlet.

For the external http service using `HttpStorageManager`:
```properties
org.opencadc.tap.tmp.HttpStorageManager.baseURL = {base URL for result files}
# specify implementation in case the DelegatingStorageManager is used
org.opencadc.tap.tmp.StorageManager = org.opencadc.tap.tmp.HttpStorageManager

# HttpStorageManager config
org.opencadc.tap.tmp.HttpStorageManager.baseURL = {base URL for tmp files}
org.opencadc.tap.tmp.HttpStorageManager.certificate = {certificate file name}
```
For the HttpStorageManager, the result will be PUT to that same URL and requires
an X509 client certificate to authenticate. The certificate is located in
{user.home}/.ssl/{certificate file name}.

The HttpStorageManager ResultStore implementation supports a user-specified job
parameter `DEST={uri}` which will direct output of async queries to the specified
URI. This can be a VOSpace URI (`vos://{authority}~{service}/{path}`) or an `https`
URL that accepts PUT and supports GET. When DEST is used, the caller's credentials
are used instead of the configured certificate.
In addition to the configured result storage, the HttpStorageManager ResultStore
implementation supports a user-specified job parameter `DEST={uri}` which will direct
output of async queries to the specified URI. This can be a VOSpace URI
(`vos://{authority}~{service}/{path}`) or an `https` URL that accepts PUT and
supports GET. When DEST is used, the caller's credentials are used instead of the
configured certificate.

In both cases, result files will be retrievable from {baseURL}/{result_filename}
(unless DEST was used).


### Enable storage
To enable temporary uploads to disk, configure the `InlineContentHandler` in both the
TAP-sync and TAP-async servlets to load the one of the plugin classes:
To enable temporary uploads to disk, configure the `InlineContentHandler` in both
the TAP-sync and TAP-async servlets to load one of the plugin classes:

In the `web.xml`:

Expand All @@ -57,7 +69,7 @@ In the `web.xml`:
</servlet>
```

The ResultStore implementation is configured in the TAP service's
The `ResultStore` implementation is configured in the TAP service's
`PluginFactory.properties`, e.g.:

```properties
Expand Down
4 changes: 2 additions & 2 deletions cadc-tap-tmp/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ apply from: '../opencadc.gradle'
sourceCompatibility = 1.8

group = 'org.opencadc'
version = '1.1.1'
version = '1.1.5'

description = 'OpenCADC TAP-1.1 temporary on-disk upload storage library'
def git_url = 'https://github.com/opencadc/tap'
Expand All @@ -26,7 +26,7 @@ dependencies {
implementation 'org.opencadc:cadc-tap-server:[1.1.17,1.2.0)'
implementation 'org.opencadc:cadc-uws:[1.0,)'
implementation 'org.opencadc:cadc-uws-server:[1.2.12,1.3)'
implementation 'org.opencadc:cadc-vos:[1.2,)'
implementation 'org.opencadc:cadc-vos:[1.2,2.0)'

testImplementation 'junit:junit:[4.12,5.0)'
}
Loading

0 comments on commit 63cffbe

Please sign in to comment.