From fd708fc01b0f99ec8781d27395be20a122e69bb5 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 25 Apr 2023 20:53:13 -0700 Subject: [PATCH 01/23] youcat: initial transfer of code from CADC refactor into org.opencadc.youcat package change dependency on cadc-access-control to cadc-gms --- youcat/Dockerfile | 3 + youcat/README.md | 35 +- youcat/build.gradle | 67 ++ .../opencadc/youcat/AbstractTablesTest.java | 326 +++++++ .../org/opencadc/youcat/AuthQueryTest.java | 321 ++++++ .../opencadc/youcat/CatAsyncErrorTest.java | 107 ++ .../opencadc/youcat/CatAsyncQueryTest.java | 107 ++ .../opencadc/youcat/CatAsyncUploadTest.java | 70 ++ .../org/opencadc/youcat/CatSyncErrorTest.java | 107 ++ .../org/opencadc/youcat/CatSyncQueryTest.java | 105 ++ .../opencadc/youcat/CatSyncUploadTest.java | 31 + .../java/org/opencadc/youcat/Constants.java | 84 ++ .../org/opencadc/youcat/CreateTableTest.java | 367 +++++++ .../opencadc/youcat/LoadTableDataTest.java | 710 ++++++++++++++ .../org/opencadc/youcat/PermissionsTest.java | 913 ++++++++++++++++++ .../youcat/RegistryClientLookupTest.java | 363 +++++++ .../org/opencadc/youcat/TokenAccessTest.java | 362 +++++++ .../opencadc/youcat/VosiAvailabilityTest.java | 95 ++ .../opencadc/youcat/VosiCapabilitiesTest.java | 127 +++ .../org/opencadc/youcat/VosiTablesTest.java | 146 +++ .../resources/AsyncErrorTest.foo.properties | 1 + .../resources/AsyncErrorTest.pi.properties | 1 + .../AsyncResultTest-ts-columns.properties | 1 + .../SyncErrorTest-ATTEMPT-DELETE.properties | 2 + .../SyncErrorTest-ATTEMPT-INSERT.properties | 2 + .../SyncErrorTest-ATTEMPT-UPDATE.properties | 2 + .../SyncErrorTest-BAD-LANG.properties | 2 + .../SyncErrorTest-BAD-QUERY.properties | 2 + .../SyncErrorTest-NO-LANG.properties | 1 + .../SyncErrorTest-NO-QUERY.properties | 1 + .../SyncErrorTest-NO-SUCH-TABLE.properties | 2 + .../resources/SyncErrorTest.foo.properties | 2 + .../resources/SyncErrorTest.pi.properties | 2 + .../resources/SyncErrorTest.rand.properties | 2 + .../resources/SyncErrorTest.range.properties | 2 + .../SyncErrorTest.truncate.properties | 2 + .../SyncResultTest-ts-columns.properties | 2 + .../SyncResultTest-ts-keycolumns.properties | 2 + .../SyncResultTest-ts-keys.properties | 2 + .../SyncResultTest-ts-schemas.properties | 2 + .../SyncResultTest-ts-tables-fqcn.properties | 2 + .../SyncResultTest-ts-tables.properties | 2 + .../src/intTest/resources/TAPUploadTest-1.xml | 26 + .../src/intTest/resources/TAPUploadTest-2.xml | 30 + .../opencadc/youcat/CatalogQueryRunner.java | 106 ++ .../opencadc/youcat/CatalogTapService.java | 226 +++++ .../youcat/DataSourceProviderImpl.java | 133 +++ .../org/opencadc/youcat/QueryJobManager.java | 108 +++ .../youcat/TableUpdateJobManager.java | 107 ++ .../youcat/TableUpdateRunnerImpl.java | 98 ++ .../opencadc/youcat/tap/AdqlQueryImpl.java | 143 +++ .../youcat/tap/FormatFactoryImpl.java | 137 +++ .../youcat/tap/MaxRecValidatorImpl.java | 169 ++++ .../opencadc/youcat/tap/TapSchemaDAOImpl.java | 138 +++ .../youcat/tap/UploadManagerImpl.java | 109 +++ .../main/resources/PluginFactory.properties | 25 + youcat/src/main/webapp/META-INF/context.xml | 41 + youcat/src/main/webapp/WEB-INF/web.xml | 298 ++++++ youcat/src/main/webapp/capabilities.xml | 173 ++++ youcat/src/main/webapp/index.html | 165 ++++ youcat/src/main/webapp/service.json | 772 +++++++++++++++ .../org/opencadc/youcat/TestDataSource.java | 70 ++ .../youcat/tap/AdqlQueryImplTest.java | 208 ++++ .../youcat/tap/MaxRecValidatorImplTest.java | 160 +++ .../resources/config/cadc-registry.properties | 11 + 65 files changed, 7936 insertions(+), 2 deletions(-) create mode 100644 youcat/Dockerfile create mode 100644 youcat/build.gradle create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/AbstractTablesTest.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/AuthQueryTest.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/CatAsyncErrorTest.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/CatAsyncQueryTest.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/CatAsyncUploadTest.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/CatSyncErrorTest.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/CatSyncQueryTest.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/CatSyncUploadTest.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/Constants.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/CreateTableTest.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/LoadTableDataTest.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/PermissionsTest.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/RegistryClientLookupTest.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/TokenAccessTest.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/VosiAvailabilityTest.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/VosiCapabilitiesTest.java create mode 100644 youcat/src/intTest/java/org/opencadc/youcat/VosiTablesTest.java create mode 120000 youcat/src/intTest/resources/AsyncErrorTest.foo.properties create mode 120000 youcat/src/intTest/resources/AsyncErrorTest.pi.properties create mode 120000 youcat/src/intTest/resources/AsyncResultTest-ts-columns.properties create mode 100644 youcat/src/intTest/resources/SyncErrorTest-ATTEMPT-DELETE.properties create mode 100644 youcat/src/intTest/resources/SyncErrorTest-ATTEMPT-INSERT.properties create mode 100644 youcat/src/intTest/resources/SyncErrorTest-ATTEMPT-UPDATE.properties create mode 100644 youcat/src/intTest/resources/SyncErrorTest-BAD-LANG.properties create mode 100644 youcat/src/intTest/resources/SyncErrorTest-BAD-QUERY.properties create mode 100644 youcat/src/intTest/resources/SyncErrorTest-NO-LANG.properties create mode 100644 youcat/src/intTest/resources/SyncErrorTest-NO-QUERY.properties create mode 100644 youcat/src/intTest/resources/SyncErrorTest-NO-SUCH-TABLE.properties create mode 100644 youcat/src/intTest/resources/SyncErrorTest.foo.properties create mode 100644 youcat/src/intTest/resources/SyncErrorTest.pi.properties create mode 100644 youcat/src/intTest/resources/SyncErrorTest.rand.properties create mode 100644 youcat/src/intTest/resources/SyncErrorTest.range.properties create mode 100644 youcat/src/intTest/resources/SyncErrorTest.truncate.properties create mode 100644 youcat/src/intTest/resources/SyncResultTest-ts-columns.properties create mode 100644 youcat/src/intTest/resources/SyncResultTest-ts-keycolumns.properties create mode 100644 youcat/src/intTest/resources/SyncResultTest-ts-keys.properties create mode 100644 youcat/src/intTest/resources/SyncResultTest-ts-schemas.properties create mode 100644 youcat/src/intTest/resources/SyncResultTest-ts-tables-fqcn.properties create mode 100644 youcat/src/intTest/resources/SyncResultTest-ts-tables.properties create mode 100644 youcat/src/intTest/resources/TAPUploadTest-1.xml create mode 100644 youcat/src/intTest/resources/TAPUploadTest-2.xml create mode 100644 youcat/src/main/java/org/opencadc/youcat/CatalogQueryRunner.java create mode 100644 youcat/src/main/java/org/opencadc/youcat/CatalogTapService.java create mode 100644 youcat/src/main/java/org/opencadc/youcat/DataSourceProviderImpl.java create mode 100644 youcat/src/main/java/org/opencadc/youcat/QueryJobManager.java create mode 100644 youcat/src/main/java/org/opencadc/youcat/TableUpdateJobManager.java create mode 100644 youcat/src/main/java/org/opencadc/youcat/TableUpdateRunnerImpl.java create mode 100644 youcat/src/main/java/org/opencadc/youcat/tap/AdqlQueryImpl.java create mode 100644 youcat/src/main/java/org/opencadc/youcat/tap/FormatFactoryImpl.java create mode 100644 youcat/src/main/java/org/opencadc/youcat/tap/MaxRecValidatorImpl.java create mode 100644 youcat/src/main/java/org/opencadc/youcat/tap/TapSchemaDAOImpl.java create mode 100644 youcat/src/main/java/org/opencadc/youcat/tap/UploadManagerImpl.java create mode 100644 youcat/src/main/resources/PluginFactory.properties create mode 100644 youcat/src/main/webapp/META-INF/context.xml create mode 100644 youcat/src/main/webapp/WEB-INF/web.xml create mode 100644 youcat/src/main/webapp/capabilities.xml create mode 100644 youcat/src/main/webapp/index.html create mode 100644 youcat/src/main/webapp/service.json create mode 100644 youcat/src/test/java/org/opencadc/youcat/TestDataSource.java create mode 100644 youcat/src/test/java/org/opencadc/youcat/tap/AdqlQueryImplTest.java create mode 100644 youcat/src/test/java/org/opencadc/youcat/tap/MaxRecValidatorImplTest.java create mode 100644 youcat/src/test/resources/config/cadc-registry.properties diff --git a/youcat/Dockerfile b/youcat/Dockerfile new file mode 100644 index 00000000..f1d9d4d4 --- /dev/null +++ b/youcat/Dockerfile @@ -0,0 +1,3 @@ +FROM images.opencadc.org/library/cadc-tomcat:1 + +COPY build/libs/youcat.war /usr/share/tomcat/webapps diff --git a/youcat/README.md b/youcat/README.md index 20ab2185..b476a5cd 100644 --- a/youcat/README.md +++ b/youcat/README.md @@ -4,14 +4,45 @@ YouCat is a complete TAP service implementation that supports TAP-1.1 plus CADC/CANFAR extensions to enable users to create their own tables and load data into them. -## TODO +## configuration + +The following configuration files must be available in the `/config` directory. + +### catalina.properties + +This file contains java system properties to configure the tomcat server and some +of the java libraries used in the service. + +See cadc-tomcat +for system properties related to the deployment environment. + +See cadc-util +for common system properties. + +`baldur` includes multiple IdentityManager implementations to support authenticated access: + - See cadc-access-control-identity for CADC access-control system support. + + - See cadc-gms for OIDC token support. + +### cadc-registry.properties + +See cadc-registry. + +## youcat.properties + +As hard-coded behaviours of `youcat` are extracted from the build and made configurable, +the configuration options will usually be in this file (see **development plans** below). + +The `youcat` implementation is currently hard-coded to use a PostgreSQL backend database. + +## development plans Make all aspects of the service configurable at runtime so deployers can use the standard image published by CADC: - move some code from youcat into appropriate opencadc library -- extract PluginFactory.properties from inside war +- extract PluginFactory.properties from inside war (cadc-tap-server) - document database requirements and configuration diff --git a/youcat/build.gradle b/youcat/build.gradle new file mode 100644 index 00000000..5ab98d0a --- /dev/null +++ b/youcat/build.gradle @@ -0,0 +1,67 @@ +plugins { + id 'war' + id 'maven' + id 'maven-publish' + id 'checkstyle' +} + +repositories { + mavenCentral() + mavenLocal() +} + +apply from: '../opencadc.gradle' + +sourceCompatibility = 1.8 + +group = 'ca.nrc.cadc' + +war { + // Include the swagger-ui so that /cat provides the cat API documentation + from(System.getenv('RPS') + '/resources/') { + include 'swagger-ui/' + } +} + +dependencies { + providedCompile 'javax.servlet:javax.servlet-api:[3.1.0,)' + providedCompile 'org.postgresql:postgresql:[42.2,43.0)' + + compile 'org.opencadc:cadc-util:[1.6.1,2.0)' + runtime 'org.opencadc:cadc-log:[1.1.1,)' + compile 'org.opencadc:cadc-rest:[1.3.10,)' + compile 'org.opencadc:cadc-dali:[1.2.12,)' + compile 'org.opencadc:cadc-adql:[1.1.8,)' + compile 'org.opencadc:cadc-uws-server:[1.2.8,)' + compile 'org.opencadc:cadc-tap-schema:[1.1.24,)' + compile 'org.opencadc:cadc-tap-server:[1.1.15,)' + compile 'org.opencadc:cadc-tap-server-pg:[1.0.5,)' + compile 'org.opencadc:cadc-adql:[1.1.7,)' + compile 'org.opencadc:cadc-vosi:[1.4.3,2.0)' + compile 'org.opencadc:cadc-uws:[1.0.2,)' + compile 'org.opencadc:cadc-uws-server:[1.2.6,)' + compile 'org.opencadc:cadc-tap-tmp:[1.1,)' + compile 'org.opencadc:cadc-gms:[1.0,)' + runtime 'org.opencadc:cadc-registry:[1.5.10,)' + + runtime 'org.opencadc:cadc-gms:[1.0.4,2.0)' + runtime 'org.opencadc:cadc-access-control-identity:[1.1.0,)' + runtime 'org.apache.ant:ant:1.9.4' + runtime 'org.opencadc:nom-tam-fits:[1.16.0,2.0)' + //runtime 'uk.ac.starlink:jcdf:[1.2.3,2.0' + //runtime 'uk.ac.starlink:stil:[4.0,5.0)' + + testCompile 'junit:junit:[4.0,)' + intTestCompile 'org.opencadc:cadc-tap:[1.0,)' + intTestCompile 'org.opencadc:cadc-test-uws:[1.1.1,)' + intTestCompile 'org.opencadc:cadc-test-vosi:[1.0.11,)' + intTestCompile 'org.opencadc:cadc-test-tap:[1.1,)' + +} + +configurations { + runtime.exclude group: 'javax.servlet' + runtime.exclude group: 'net.sourceforge.jtds' + runtime.exclude group: 'org.postgresql' + runtime.exclude group: 'org.restlet.jee' +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/AbstractTablesTest.java b/youcat/src/intTest/java/org/opencadc/youcat/AbstractTablesTest.java new file mode 100644 index 00000000..f748a0ab --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/AbstractTablesTest.java @@ -0,0 +1,326 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2019. (c) 2019. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.youcat; + + +import ca.nrc.cadc.auth.AuthMethod; +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.auth.RunnableAction; +import ca.nrc.cadc.auth.SSLUtil; +import ca.nrc.cadc.net.FileContent; +import ca.nrc.cadc.net.HttpDelete; +import ca.nrc.cadc.net.HttpDownload; +import ca.nrc.cadc.net.HttpPost; +import ca.nrc.cadc.net.HttpUpload; +import ca.nrc.cadc.net.OutputStreamWrapper; +import ca.nrc.cadc.reg.Standards; +import ca.nrc.cadc.reg.client.RegistryClient; +import ca.nrc.cadc.tap.schema.ColumnDesc; +import ca.nrc.cadc.tap.schema.TableDesc; +import ca.nrc.cadc.tap.schema.TapDataType; +import ca.nrc.cadc.tap.schema.TapPermissions; +import ca.nrc.cadc.util.FileUtil; +import ca.nrc.cadc.util.Log4jInit; +import ca.nrc.cadc.uws.ExecutionPhase; +import ca.nrc.cadc.uws.Job; +import ca.nrc.cadc.uws.JobReader; +import ca.nrc.cadc.vosi.TableWriter; +import ca.nrc.cadc.vosi.actions.TableDescHandler; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.StringReader; +import java.io.StringWriter; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.charset.Charset; +import java.util.Map; +import java.util.TreeMap; +import javax.security.auth.Subject; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.opencadc.tap.TapClient; +import org.postgresql.shaded.com.ongres.scram.common.util.StringWritable; + +/** + * base class with common setup and methods. + * + * @author pdowler + */ +abstract class AbstractTablesTest { + private static final Logger log = Logger.getLogger(AbstractTablesTest.class); + + static { + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.tap", Level.INFO); + } + + static String VALID_TEST_GROUP = "ivo://cadc.nrc.ca/gms?YouCat-ReadWrite"; + + Subject anon; + Subject schemaOwner; + Subject subjectWithGroups; + + URL anonQueryURL; + URL certQueryURL; + URL anonTablesURL; + URL certTablesURL; + URL certUpdateURL; + URL certLoadURL; + URL permsURL; + + AbstractTablesTest() { + try { + File cf = FileUtil.getFileFromResource("x509_CADCAuthtest1.pem", AbstractTablesTest.class); + schemaOwner = SSLUtil.createSubject(cf); + anon = AuthenticationUtil.getAnonSubject(); + log.debug("created schemaOwner: " + schemaOwner); + + cf = FileUtil.getFileFromResource("x509_CADCAuthtest2.pem", AbstractTablesTest.class); + subjectWithGroups = SSLUtil.createSubject(cf); + log.debug("created subjectWithGroups: " + subjectWithGroups); + + try { + RegistryClient reg = new RegistryClient(); + TapClient tc = new TapClient(Constants.RESOURCE_ID); + anonQueryURL = tc.getSyncURL(Standards.SECURITY_METHOD_ANON); + certQueryURL = tc.getSyncURL(Standards.SECURITY_METHOD_CERT); + anonTablesURL = reg.getServiceURL(Constants.RESOURCE_ID, Standards.VOSI_TABLES_11, AuthMethod.ANON); + certTablesURL = reg.getServiceURL(Constants.RESOURCE_ID, Standards.VOSI_TABLES_11, AuthMethod.CERT); + certUpdateURL = reg.getServiceURL(Constants.RESOURCE_ID, Standards.PROTO_TABLE_UPDATE_ASYNC, AuthMethod.CERT); + certLoadURL = reg.getServiceURL(Constants.RESOURCE_ID, Standards.PROTO_TABLE_LOAD_SYNC, AuthMethod.CERT); + permsURL = reg.getServiceURL(Constants.RESOURCE_ID, Standards.PROTO_TABLE_PERMISSIONS, AuthMethod.CERT); + } catch (Exception ex) { + log.error("TEST SETUP BUG: failed to find TAP URL", ex); + } + } catch (Throwable t) { + throw new RuntimeException("TEST SETUP FAILED", t); + } + } + + void doDelete(Subject subject, String testTable, boolean fnf) throws Exception { + // delete + URL tableURL = new URL(certTablesURL.toExternalForm() + "/" + testTable); + HttpDelete del = new HttpDelete(tableURL, false); + log.info("doDelete: " + tableURL); + Subject.doAs(subject, new RunnableAction(del)); + if (fnf) { + return; + } + + // check if it worked + log.info("doDelete: verifying..."); + if (del.getThrowable() != null && del.getThrowable() instanceof Exception) { + throw (Exception) del.getThrowable(); + } + + URL getTableURL = new URL(certTablesURL.toExternalForm() + "/" + testTable); + HttpDownload check = new HttpDownload(getTableURL, new ByteArrayOutputStream()); + Subject.doAs(subject, new RunnableAction(check)); + Assert.assertEquals("table deleted", 404, check.getResponseCode()); + } + + TableDesc doCreateTable(Subject subject, String tableName) throws Exception { + // cleanup just in case + doDelete(subject, tableName, true); + + final TableDesc orig = new TableDesc("cadcauthtest1", tableName); + orig.description = "created by intTest"; + orig.tableType = TableDesc.TableType.TABLE; + orig.tableIndex = 1; + orig.getColumnDescs().add(new ColumnDesc(tableName, "c0", TapDataType.STRING)); + orig.getColumnDescs().add(new ColumnDesc(tableName, "c1", TapDataType.SHORT)); + orig.getColumnDescs().add(new ColumnDesc(tableName, "c2", TapDataType.INTEGER)); + orig.getColumnDescs().add(new ColumnDesc(tableName, "c3", TapDataType.LONG)); + orig.getColumnDescs().add(new ColumnDesc(tableName, "c4", TapDataType.FLOAT)); + orig.getColumnDescs().add(new ColumnDesc(tableName, "c5", TapDataType.DOUBLE)); + orig.getColumnDescs().add(new ColumnDesc(tableName, "c6", TapDataType.TIMESTAMP)); + orig.getColumnDescs().add(new ColumnDesc(tableName, "e7", TapDataType.INTERVAL)); + orig.getColumnDescs().add(new ColumnDesc(tableName, "e8", TapDataType.POINT)); + orig.getColumnDescs().add(new ColumnDesc(tableName, "e9", TapDataType.CIRCLE)); + orig.getColumnDescs().add(new ColumnDesc(tableName, "e10", TapDataType.POLYGON)); + // arrays + orig.getColumnDescs().add(new ColumnDesc(tableName, "a11", new TapDataType("short", "*", null))); + orig.getColumnDescs().add(new ColumnDesc(tableName, "a12", new TapDataType("int", "*", null))); + orig.getColumnDescs().add(new ColumnDesc(tableName, "a13", new TapDataType("long", "*", null))); + orig.getColumnDescs().add(new ColumnDesc(tableName, "a14", new TapDataType("float", "*", null))); + orig.getColumnDescs().add(new ColumnDesc(tableName, "a15", new TapDataType("double", "*", null))); + + // create + URL tableURL = new URL(certTablesURL.toExternalForm() + "/" + tableName); + TableWriter w = new TableWriter(); + StringWriter sw = new StringWriter(); + w.write(orig, sw); + log.info("VOSI-table description:\n" + sw.toString()); + + OutputStreamWrapper src = new OutputStreamWrapper() { + @Override + public void write(OutputStream out) throws IOException { + TableWriter w = new TableWriter(); + w.write(orig, new OutputStreamWriter(out)); + } + }; + HttpUpload put = new HttpUpload(src, tableURL); + put.setContentType(TableDescHandler.VOSI_TABLE_TYPE); + log.info("doCreateTable: " + tableURL); + Subject.doAs(subject, new RunnableAction(put)); + log.info("doCreateTable: " + put.getResponseCode()); + if (put.getThrowable() != null && put.getThrowable() instanceof Exception) { + throw (Exception) put.getThrowable(); + } + Assert.assertEquals("response code", 200, put.getResponseCode()); + return orig; + } + + void doCreateIndex(Subject subject, String tableName, String indexCol, boolean unique, ExecutionPhase expected, String emsg) throws Exception { + Assert.assertNotNull("found async table-update URL", certUpdateURL); + + // create job + Map params = new TreeMap(); + params.put("table", tableName); + params.put("index", indexCol); + params.put("unique", Boolean.toString(unique)); + HttpPost post = new HttpPost(certUpdateURL, params, false); + Subject.doAs(subject, new RunnableAction(post)); + Assert.assertNull("throwable", post.getThrowable()); + Assert.assertEquals("response code", 303, post.getResponseCode()); + final URL jobURL = post.getRedirectURL(); + Assert.assertNotNull("jobURL", jobURL); + log.info("create index job: " + jobURL); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpDownload get = new HttpDownload(jobURL, bos); + Subject.doAs(subject, new RunnableAction(get)); + Assert.assertNull("throwable", get.getThrowable()); + Assert.assertEquals("response code", 200, get.getResponseCode()); + String xml = bos.toString(); + log.debug("create index job:\n" + xml); + + // execute job + params.clear(); + params.put("PHASE", "RUN"); + final URL phaseURL = new URL(jobURL.toExternalForm() + "/phase"); + post = new HttpPost(phaseURL, params, false); + Subject.doAs(subject, new RunnableAction(post)); + Assert.assertNull("throwable", post.getThrowable()); + + // wait for completion + bos.reset(); + final URL blockURL = new URL(jobURL.toExternalForm() + "?WAIT=60"); + HttpDownload block = new HttpDownload(blockURL, bos); + Subject.doAs(subject, new RunnableAction(block)); + Assert.assertNull("throwable", block.getThrowable()); + Assert.assertEquals("response code", 200, block.getResponseCode()); + String xml2 = bos.toString(); + log.debug("final job state:\n" + xml2); + + JobReader r = new JobReader(); + Job end = r.read(new StringReader(xml2)); + Assert.assertEquals("final job state", expected, end.getExecutionPhase()); + if (emsg != null) { + Assert.assertTrue("error message", end.getErrorSummary().getSummaryMessage().startsWith(emsg)); + } + } + + protected void clearSchemaPerms() throws MalformedURLException { + TapPermissions tp = new TapPermissions(); + tp.isPublic = false; + setPerms(schemaOwner, "cadcauthtest1", tp, 200); + } + + protected void setPerms(Subject subject, String name, TapPermissions tp, int expectedCode) throws MalformedURLException { + + StringBuilder perms = new StringBuilder(); + if (tp.isPublic != null) { + perms.append("public=").append(Boolean.toString(tp.isPublic)).append("\n"); + } else { + perms.append("public=false\n"); + } + if (tp.readGroup != null) { + perms.append("r-group=").append(tp.readGroup.getURI()).append("\n"); + } else { + perms.append("r-group=\n"); + } + if (tp.readWriteGroup != null) { + perms.append("rw-group=").append(tp.readWriteGroup.getURI()).append("\n"); + } else { + perms.append("rw-group=\n"); + } + + log.info("Setting perms: " + perms); + + FileContent content = new FileContent(perms.toString(), "text/plain", Charset.forName("utf-8")); + URL schemaURL = new URL(permsURL.toString() + "/" + name); + HttpPost post = new HttpPost(schemaURL, content, false); + Subject.doAs(subject, new RunnableAction(post)); + Assert.assertEquals(expectedCode, post.getResponseCode()); + + } +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/AuthQueryTest.java b/youcat/src/intTest/java/org/opencadc/youcat/AuthQueryTest.java new file mode 100644 index 00000000..6a3ebbca --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/AuthQueryTest.java @@ -0,0 +1,321 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2019. (c) 2019. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +* $Revision: 5 $ +* +************************************************************************ + */ + +package org.opencadc.youcat; + +import ca.nrc.cadc.auth.AuthMethod; +import ca.nrc.cadc.auth.RunnableAction; +import ca.nrc.cadc.auth.SSLUtil; +import ca.nrc.cadc.dali.tables.votable.VOTableDocument; +import ca.nrc.cadc.dali.tables.votable.VOTableInfo; +import ca.nrc.cadc.dali.tables.votable.VOTableReader; +import ca.nrc.cadc.dali.tables.votable.VOTableResource; +import ca.nrc.cadc.net.HttpDownload; +import ca.nrc.cadc.net.HttpGet; +import ca.nrc.cadc.net.HttpPost; +import ca.nrc.cadc.reg.Standards; +import ca.nrc.cadc.util.FileUtil; +import ca.nrc.cadc.util.Log4jInit; +import ca.nrc.cadc.uws.ExecutionPhase; +import ca.nrc.cadc.uws.Job; +import ca.nrc.cadc.uws.JobListReader; +import ca.nrc.cadc.uws.JobReader; +import ca.nrc.cadc.uws.JobRef; +import ca.nrc.cadc.uws.Result; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.StringReader; +import java.net.URL; +import java.security.Principal; +import java.security.PrivilegedExceptionAction; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import javax.security.auth.Subject; +import javax.security.auth.x500.X500Principal; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; +import org.opencadc.tap.TapClient; + +/** + * Half-decent test that authenticated queries work. + * + * @author pdowler + */ +public class AuthQueryTest { + + private static final Logger log = Logger.getLogger(AuthQueryTest.class); + + static final Subject subjectWithGroups; + static final Principal userWithGroups; + static URL syncCertURL; + static URL asyncCertURL; + + static { + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + + // need to read cert so we have creds to make the fake GMS call + File cf = FileUtil.getFileFromResource("x509_CADCAuthtest1.pem", AuthQueryTest.class); + subjectWithGroups = SSLUtil.createSubject(cf); + userWithGroups = subjectWithGroups.getPrincipals(X500Principal.class).iterator().next(); + log.debug("created subjectWithGroups: " + subjectWithGroups); + + try { + TapClient tc = new TapClient(Constants.RESOURCE_ID); + syncCertURL = tc.getSyncURL(Standards.SECURITY_METHOD_CERT); + asyncCertURL = tc.getAsyncURL(Standards.SECURITY_METHOD_CERT); + } catch (Exception ex) { + log.error("TEST SETUP BUG: failed to find TAP URL", ex); + } + } + + static class SyncQueryAction implements PrivilegedExceptionAction { + + private URL url; + private Map params; + + public SyncQueryAction(URL url, Map params) { + this.url = url; + this.params = params; + } + + @Override + public String run() + throws Exception { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpPost doit = new HttpPost(url, params, out); + doit.run(); + + if (doit.getThrowable() != null) { + log.error("Post failed", doit.getThrowable()); + Assert.fail("exception on post: " + doit.getThrowable()); + } + + int code = doit.getResponseCode(); + Assert.assertEquals(200, code); + + String contentType = doit.getContentType(); + String result = out.toString(); + + log.debug("contentType: " + contentType); + log.debug("respnse:\n" + result); + + Assert.assertEquals("application/x-votable+xml", contentType); + + return result; + } + + } + + static class AsyncQueryAction implements PrivilegedExceptionAction { + + private URL url; + private Map params; + + public AsyncQueryAction(URL url, Map params) { + this.url = url; + this.params = params; + } + + @Override + public Job run() + throws Exception { + HttpPost doit = new HttpPost(url, params, false); + doit.run(); + + if (doit.getThrowable() != null) { + log.error("Post failed", doit.getThrowable()); + Assert.fail("exception on post: " + doit.getThrowable()); + } + + int code = doit.getResponseCode(); + Assert.assertEquals(303, code); + + URL jobURL = doit.getRedirectURL(); + + // exec the job + URL phaseURL = new URL(jobURL.toString() + "/phase"); + Map nextParams = new HashMap(); + nextParams.put("PHASE", "RUN"); + doit = new HttpPost(phaseURL, nextParams, false); + doit.run(); + + if (doit.getThrowable() != null) { + log.error("Post failed", doit.getThrowable()); + Assert.fail("exception on post: " + doit.getThrowable()); + } + + JobReader jr = new JobReader(); + Job job = null; + URL waitURL = new URL(jobURL.toExternalForm() + "?WAIT=30"); + boolean done = false; + while (!done) { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpDownload w = new HttpDownload(waitURL, out); + w.run(); + job = jr.read(new ByteArrayInputStream(out.toByteArray())); + ExecutionPhase ep = job.getExecutionPhase(); + done = !ep.isActive(); + } + Assert.assertNotNull("job", job); + + return job; + } + + } + + @Test + public void testAnonQueryPrivateTable() { + try { + // select a row from a proprietary table in a publicly readable schema + String adql = "SELECT TOP 1 * from cfht.luausdss"; + + Map params = new TreeMap(); + params.put("LANG", "ADQL"); + params.put("QUERY", adql); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpPost doit = new HttpPost(syncCertURL, params, bos); + doit.setFollowRedirects(true); + doit.run(); + + Assert.assertEquals(400, doit.getResponseCode()); + Assert.assertNotNull(doit.getThrowable()); + + VOTableReader r = new VOTableReader(); + VOTableDocument doc = r.read(doit.getThrowable().getMessage()); + VOTableResource vr = doc.getResourceByType("results"); + VOTableInfo queryStatus = null; + for (VOTableInfo vi : vr.getInfos()) { + if ("QUERY_STATUS".equals(vi.getName())) { + queryStatus = vi; + } + } + Assert.assertNotNull(queryStatus); + Assert.assertEquals("ERROR", queryStatus.getValue()); + Assert.assertNotNull(queryStatus.content); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testVOSAuthQuery() { + try { + String adql = "SELECT top 1 * from tap_schema.tables"; + + Map params = new TreeMap(); + params.put("LANG", "ADQL"); + params.put("QUERY", adql); + params.put("DEST", "vos://cadc.nrc.ca~vault/CADCAuthtest1/test/youcat-testVOSAuthQuery"); + + Job job = Subject.doAs(subjectWithGroups, new AsyncQueryAction(asyncCertURL, params)); + log.debug("Job phase: " + job.getExecutionPhase()); + log.info("jobID: " + job.getID() + " phase: " + job.getExecutionPhase() + " error: " + job.getErrorSummary()); + Assert.assertTrue("job completed", job.getExecutionPhase().equals(ExecutionPhase.COMPLETED)); + for (Result r : job.getResultsList()) { + log.info(r.getName() + ": " + r.getURI()); + if ("result".equals(r.getName())) { // spec result name since TAP-1.0 + Assert.assertTrue("result stored in vault", r.getURI().toASCIIString().contains("vault")); + } + } + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testAuthJobList() { + try { + URL listURL = new URL(asyncCertURL.toExternalForm() + "?LAST=5"); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpGet get = new HttpGet(asyncCertURL, bos); + Subject.doAs(subjectWithGroups, new RunnableAction(get)); + log.info("code: " + get.getResponseCode() + " error: " + get.getThrowable()); + Assert.assertEquals(200, get.getResponseCode()); + Assert.assertNull(get.getThrowable()); + String xml = bos.toString("UTF-8"); + log.debug("job list:\n" + xml); + JobListReader r = new JobListReader(); + List jobs = r.read(new StringReader(xml)); + for (JobRef j : jobs) { + log.info("job: " + j); + } + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/CatAsyncErrorTest.java b/youcat/src/intTest/java/org/opencadc/youcat/CatAsyncErrorTest.java new file mode 100644 index 00000000..38b9b5c6 --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/CatAsyncErrorTest.java @@ -0,0 +1,107 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2011. (c) 2011. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +* $Revision: 5 $ +* +************************************************************************ +*/ + +package org.opencadc.youcat; + + +import ca.nrc.cadc.tap.integration.TapAsyncErrorTest; +import ca.nrc.cadc.util.FileUtil; +import ca.nrc.cadc.util.Log4jInit; +import java.io.File; +import java.net.URI; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +public class CatAsyncErrorTest extends TapAsyncErrorTest +{ + private static final Logger log = Logger.getLogger(CatAsyncErrorTest.class); + + static { + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.tap", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.conformance.uws2", Level.INFO); + } + + public CatAsyncErrorTest() + { + super(Constants.RESOURCE_ID); + + // re-use SyncResultTest files + File testFile = FileUtil.getFileFromResource("AsyncErrorTest.foo.properties", CatSyncQueryTest.class); + if (testFile.exists()) + { + File testDir = testFile.getParentFile(); + super.setPropertiesDir(testDir, "AsyncErrorTest"); + } + } +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/CatAsyncQueryTest.java b/youcat/src/intTest/java/org/opencadc/youcat/CatAsyncQueryTest.java new file mode 100644 index 00000000..950058a3 --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/CatAsyncQueryTest.java @@ -0,0 +1,107 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2011. (c) 2011. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +* $Revision: 5 $ +* +************************************************************************ +*/ + +package org.opencadc.youcat; + + +import ca.nrc.cadc.tap.integration.TapAsyncQueryTest; +import ca.nrc.cadc.util.FileUtil; +import ca.nrc.cadc.util.Log4jInit; +import java.io.File; +import java.net.URI; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +public class CatAsyncQueryTest extends TapAsyncQueryTest +{ + private static final Logger log = Logger.getLogger(CatAsyncQueryTest.class); + + static { + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.tap", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.conformance.uws2", Level.INFO); + } + + public CatAsyncQueryTest() + { + super(Constants.RESOURCE_ID); + + // re-use SyncResultTest files + File testFile = FileUtil.getFileFromResource("AsyncResultTest-ts-columns.properties", CatSyncQueryTest.class); + if (testFile.exists()) + { + File testDir = testFile.getParentFile(); + super.setPropertiesDir(testDir, "AsyncResultTest"); + } + } +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/CatAsyncUploadTest.java b/youcat/src/intTest/java/org/opencadc/youcat/CatAsyncUploadTest.java new file mode 100644 index 00000000..c2f2c23e --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/CatAsyncUploadTest.java @@ -0,0 +1,70 @@ + +package org.opencadc.youcat; + +import ca.nrc.cadc.auth.AuthMethod; +import ca.nrc.cadc.auth.RunnableAction; +import ca.nrc.cadc.auth.SSLUtil; +import ca.nrc.cadc.net.HttpTransfer; +import ca.nrc.cadc.net.HttpUpload; +import ca.nrc.cadc.reg.Standards; +import ca.nrc.cadc.reg.client.RegistryClient; +import ca.nrc.cadc.tap.integration.TapAsyncUploadTest; +import ca.nrc.cadc.util.FileUtil; +import ca.nrc.cadc.util.Log4jInit; +import java.io.File; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import javax.security.auth.Subject; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +public class CatAsyncUploadTest extends TapAsyncUploadTest { + + private static final Logger log = Logger.getLogger(CatAsyncUploadTest.class); + + // prod CADC minoc + static final URI STORAGE_RESOURCE_ID = URI.create("ivo://cadc.nrc.ca/cadc/minoc"); + + static { + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.tap", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.conformance.uws2", Level.INFO); + } + + public CatAsyncUploadTest() { + super(Constants.RESOURCE_ID); + File f = FileUtil.getFileFromResource("TAPUploadTest-1.xml", CatSyncUploadTest.class); + setTestFile(f); + setTestURL(getVOTableURL(f)); + } + + static URL getVOTableURL(File f) { + try { + String tableName = "tab_" + System.currentTimeMillis(); + + File cert = FileUtil.getFileFromResource("tmpops.pem", CatAsyncUploadTest.class); + Subject subject = SSLUtil.createSubject(cert); + + RegistryClient reg = new RegistryClient(); + URL storageURL = reg.getServiceURL(STORAGE_RESOURCE_ID, Standards.SI_FILES, AuthMethod.CERT); + URL putURL = new URL(storageURL.toExternalForm() + "/tmp:youcat-int-test/" + tableName + ".xml"); + HttpUpload httpUpload = new HttpUpload(f, putURL); + httpUpload.setRequestProperty(HttpTransfer.CONTENT_TYPE, "text/xml"); + Subject.doAs(subject, new RunnableAction(httpUpload)); + + // Error during the upload, throw an exception. + if (httpUpload.getThrowable() != null) { + throw new RuntimeException("setup failed to store VOTable upload ", httpUpload.getThrowable()); + } + + return putURL; + } catch (MalformedURLException ex) { + throw new RuntimeException("BUG: failed to generate put URL", ex); + } + } +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/CatSyncErrorTest.java b/youcat/src/intTest/java/org/opencadc/youcat/CatSyncErrorTest.java new file mode 100644 index 00000000..09bf5bed --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/CatSyncErrorTest.java @@ -0,0 +1,107 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2011. (c) 2011. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +* $Revision: 5 $ +* +************************************************************************ +*/ + +package org.opencadc.youcat; + + +import ca.nrc.cadc.tap.integration.TapSyncErrorTest; +import ca.nrc.cadc.util.FileUtil; +import ca.nrc.cadc.util.Log4jInit; +import java.io.File; +import java.net.URI; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +public class CatSyncErrorTest extends TapSyncErrorTest +{ + private static final Logger log = Logger.getLogger(CatSyncErrorTest.class); + + static { + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.tap", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.conformance.uws2", Level.INFO); + } + + public CatSyncErrorTest() + { + super(Constants.RESOURCE_ID); + + // re-use SyncResultTest files + File testFile = FileUtil.getFileFromResource("SyncErrorTest-ATTEMPT-DELETE.properties", CatSyncErrorTest.class); + if (testFile.exists()) + { + File testDir = testFile.getParentFile(); + super.setPropertiesDir(testDir, "SyncErrorTest"); + } + } +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/CatSyncQueryTest.java b/youcat/src/intTest/java/org/opencadc/youcat/CatSyncQueryTest.java new file mode 100644 index 00000000..8c20e856 --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/CatSyncQueryTest.java @@ -0,0 +1,105 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2011. (c) 2011. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +* $Revision: 5 $ +* +************************************************************************ + */ + +package org.opencadc.youcat; + +import ca.nrc.cadc.tap.integration.TapSyncQueryTest; +import ca.nrc.cadc.util.FileUtil; +import ca.nrc.cadc.util.Log4jInit; +import java.io.File; +import java.net.URI; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +public class CatSyncQueryTest extends TapSyncQueryTest { + + private static final Logger log = Logger.getLogger(CatSyncQueryTest.class); + + static { + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.tap", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.conformance.uws2", Level.INFO); + } + + public CatSyncQueryTest() { + super(Constants.RESOURCE_ID); + + // re-use SyncResultTest files + File testFile = FileUtil.getFileFromResource("SyncResultTest-ts-columns.properties", CatSyncQueryTest.class); + if (testFile.exists()) { + File testDir = testFile.getParentFile(); + super.setPropertiesDir(testDir, "SyncResultTest"); + } + } + +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/CatSyncUploadTest.java b/youcat/src/intTest/java/org/opencadc/youcat/CatSyncUploadTest.java new file mode 100644 index 00000000..46ba9d42 --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/CatSyncUploadTest.java @@ -0,0 +1,31 @@ +package org.opencadc.youcat; + +import ca.nrc.cadc.tap.integration.TapSyncUploadTest; +import ca.nrc.cadc.util.FileUtil; +import ca.nrc.cadc.util.Log4jInit; +import java.io.File; +import java.net.URI; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +// NOTE: disabled until we support UPLOAD in cadc-rest based UWS async endpoint +public class CatSyncUploadTest extends TapSyncUploadTest { + private static final Logger log = Logger.getLogger(CatSyncUploadTest.class); + + static { + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.tap", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.conformance.uws2", Level.INFO); + } + + public CatSyncUploadTest() { + super(Constants.RESOURCE_ID); + File f = FileUtil.getFileFromResource("TAPUploadTest-1.xml", CatSyncUploadTest.class); + setTestFile(f); + setTestURL(CatAsyncUploadTest.getVOTableURL(f)); + } +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/Constants.java b/youcat/src/intTest/java/org/opencadc/youcat/Constants.java new file mode 100644 index 00000000..11a96d46 --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/Constants.java @@ -0,0 +1,84 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2019. (c) 2019. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.youcat; + + +import java.net.URI; + +/** + * + * @author pdowler + */ +public abstract class Constants { + + //public static final URI RESOURCE_ID = URI.create("ivo://cadc.nrc.ca/cadc/youcat"); + public static final URI RESOURCE_ID = URI.create("ivo://cadc.nrc.ca/youcat"); + + public static final URI GMS_RESOURSE_ID = URI.create("ivo://cadc.nrc.ca/gms"); + +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/CreateTableTest.java b/youcat/src/intTest/java/org/opencadc/youcat/CreateTableTest.java new file mode 100644 index 00000000..62c03793 --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/CreateTableTest.java @@ -0,0 +1,367 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2019. (c) 2019. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.youcat; + + +import ca.nrc.cadc.auth.AuthMethod; +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.auth.RunnableAction; +import ca.nrc.cadc.auth.SSLUtil; +import ca.nrc.cadc.dali.tables.TableData; +import ca.nrc.cadc.dali.tables.votable.VOTableDocument; +import ca.nrc.cadc.dali.tables.votable.VOTableField; +import ca.nrc.cadc.dali.tables.votable.VOTableReader; +import ca.nrc.cadc.dali.tables.votable.VOTableResource; +import ca.nrc.cadc.dali.tables.votable.VOTableTable; +import ca.nrc.cadc.dali.tables.votable.VOTableWriter; +import ca.nrc.cadc.net.HttpDelete; +import ca.nrc.cadc.net.HttpDownload; +import ca.nrc.cadc.net.HttpPost; +import ca.nrc.cadc.net.HttpUpload; +import ca.nrc.cadc.net.InputStreamWrapper; +import ca.nrc.cadc.net.OutputStreamWrapper; +import ca.nrc.cadc.reg.Standards; +import ca.nrc.cadc.reg.client.RegistryClient; +import ca.nrc.cadc.tap.schema.ColumnDesc; +import ca.nrc.cadc.tap.schema.TableDesc; +import ca.nrc.cadc.tap.schema.TapDataType; +import ca.nrc.cadc.tap.schema.TapPermissions; +import ca.nrc.cadc.util.FileUtil; +import ca.nrc.cadc.util.Log4jInit; +import ca.nrc.cadc.uws.ExecutionPhase; +import ca.nrc.cadc.uws.Job; +import ca.nrc.cadc.uws.JobReader; +import ca.nrc.cadc.vosi.InvalidTableSetException; +import ca.nrc.cadc.vosi.TableReader; +import ca.nrc.cadc.vosi.TableWriter; +import ca.nrc.cadc.vosi.actions.TableDescHandler; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.StringReader; +import java.net.URL; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import javax.security.auth.Subject; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; +import org.opencadc.tap.TapClient; + +/** + * + * @author pdowler + */ +public class CreateTableTest extends AbstractTablesTest { + private static final Logger log = Logger.getLogger(CreateTableTest.class); + + static { + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.tap", Level.INFO); + } + + public CreateTableTest() { + super(); + } + + private static class StreamTableReader implements InputStreamWrapper { + TableDesc td; + + @Override + public void read(InputStream in) throws IOException { + try { + TableReader r = new TableReader(false); // schema validation causes default arraysize="1" to be injected + td = r.read(in); + } catch (InvalidTableSetException ex) { + throw new RuntimeException("invalid table metadata: ", ex); + } + } + } + + private TableDesc doVosiCheck(String testTable) throws Exception { + // VOSI tables check (metadata) + URL getTableURL = new URL(anonTablesURL.toExternalForm() + "/" + testTable); + StreamTableReader isw = new StreamTableReader(); + HttpDownload get = new HttpDownload(getTableURL, isw); + log.info("doVosiCheck: " + getTableURL); + get.run(); // anon + log.info("doVosiCheck: " + get.getResponseCode()); + Assert.assertNull("throwable", get.getThrowable()); + Assert.assertEquals("response code", 200, get.getResponseCode()); + TableDesc td = isw.td; + Assert.assertNotNull(td); + return td; + } + + private VOTableTable doQueryCheck(String testTable) throws Exception { + // TAP query check (metadata and actual table exists) + String adql = "SELECT * from " + testTable; + Map params = new TreeMap(); + params.put("LANG", "ADQL"); + params.put("QUERY", adql); + log.info("doQueryCheck: " + testTable + " " + anonQueryURL); + String result = Subject.doAs(anon, new AuthQueryTest.SyncQueryAction(anonQueryURL, params)); + log.info("doQueryCheck: verifying..."); + Assert.assertNotNull(result); + VOTableReader r = new VOTableReader(); + VOTableDocument doc = r.read(result); + VOTableResource vr = doc.getResourceByType("results"); + VOTableTable vt = vr.getTable(); + Assert.assertNotNull(vt); + return vt; + } + + @Test + public void testCreateQueryDropVOSI() { + try { + clearSchemaPerms(); + TapPermissions tp = new TapPermissions(null, true, null, null); + super.setPerms(schemaOwner, "cadcauthtest1", tp, 200); + + String testTable = "cadcauthtest1.testCreateQueryDropVOSI"; + final TableDesc orig = doCreateTable(schemaOwner, testTable); + TableDesc td = doVosiCheck(testTable); + compare(orig, td); + + super.setPerms(schemaOwner, testTable, tp, 200); + + VOTableTable vt = doQueryCheck(testTable); + TableData tdata = vt.getTableData(); + Iterator> iter = tdata.iterator(); + Assert.assertFalse("no result rows", iter.hasNext()); + + // cleanup on success + doDelete(schemaOwner, testTable, false); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testCreateQueryDropVOTable() { + try { + clearSchemaPerms(); + TapPermissions tp = new TapPermissions(null, true, null, null); + super.setPerms(schemaOwner, "cadcauthtest1", tp, 200); + + String testTable = "cadcauthtest1.testCreateQueryDropVOTable"; + + // cleanup just in case + doDelete(schemaOwner, testTable, true); + + VOTableTable vtab = new VOTableTable(); + vtab.getFields().add(new VOTableField("c0", TapDataType.STRING.getDatatype(), TapDataType.STRING.arraysize)); + vtab.getFields().add(new VOTableField("c1", TapDataType.SHORT.getDatatype())); + vtab.getFields().add(new VOTableField("c2", TapDataType.INTEGER.getDatatype())); + vtab.getFields().add(new VOTableField("c3", TapDataType.LONG.getDatatype())); + vtab.getFields().add(new VOTableField("c4", TapDataType.FLOAT.getDatatype())); + vtab.getFields().add(new VOTableField("c5", TapDataType.DOUBLE.getDatatype())); + VOTableField tsf = new VOTableField("c6", TapDataType.TIMESTAMP.getDatatype(), TapDataType.TIMESTAMP.arraysize); + tsf.xtype = TapDataType.TIMESTAMP.xtype; + vtab.getFields().add(tsf); + vtab.getFields().add(new VOTableField("e7", TapDataType.INTERVAL.getDatatype())); + vtab.getFields().add(new VOTableField("e8", TapDataType.POINT.getDatatype())); + vtab.getFields().add(new VOTableField("e9", TapDataType.CIRCLE.getDatatype())); + vtab.getFields().add(new VOTableField("e10", TapDataType.POLYGON.getDatatype())); + // arrays + vtab.getFields().add(new VOTableField("a11", "short", "*")); + vtab.getFields().add(new VOTableField("a12", "int", "*")); + vtab.getFields().add(new VOTableField("a13", "long", "*")); + vtab.getFields().add(new VOTableField("a14", "float", "*")); + vtab.getFields().add(new VOTableField("a15", "double", "*")); + + VOTableResource vres = new VOTableResource("meta"); + vres.setTable(vtab); + final VOTableDocument doc = new VOTableDocument(); + doc.getResources().add(vres); + + // create + URL tableURL = new URL(certTablesURL.toExternalForm() + "/" + testTable); + OutputStreamWrapper src = new OutputStreamWrapper() { + @Override + public void write(OutputStream out) throws IOException { + VOTableWriter w = new VOTableWriter(TableDescHandler.VOTABLE_TYPE); + w.write(doc, out); + } + }; + HttpUpload put = new HttpUpload(src, tableURL); + put.setContentType(TableDescHandler.VOTABLE_TYPE); + Subject.doAs(schemaOwner, new RunnableAction(put)); + Assert.assertNull("throwable", put.getThrowable()); + Assert.assertEquals("response code", 200, put.getResponseCode()); + put = null; + + TableDesc td = doVosiCheck(testTable); + + super.setPerms(schemaOwner, testTable, tp, 200); + + VOTableTable vt = doQueryCheck(testTable); + TableData tdata = vt.getTableData(); + Iterator> iter = tdata.iterator(); + Assert.assertFalse("no result rows", iter.hasNext()); + + + doDelete(schemaOwner, testTable, false); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testCreateIndex() { + try { + clearSchemaPerms(); + TapPermissions tp = new TapPermissions(null, true, null, null); + super.setPerms(schemaOwner, "cadcauthtest1", tp, 200); + + String tableName = "cadcauthtest1.testCreateIndex"; + TableDesc td = doCreateTable(schemaOwner, tableName); + for (ColumnDesc cd : td.getColumnDescs()) { + log.info("testCreateIndex: " + cd.getColumnName()); + ExecutionPhase expected = ExecutionPhase.COMPLETED; + if (cd.getColumnName().startsWith("a")) { + expected = ExecutionPhase.ERROR; + } + doCreateIndex(schemaOwner, tableName, cd.getColumnName(), false,expected, null); + } + + // cleanup on success + doDelete(schemaOwner, td.getTableName(), false); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testCreateUniqueIndex() { + try { + clearSchemaPerms(); + TapPermissions tp = new TapPermissions(null, true, null, null); + super.setPerms(schemaOwner, "cadcauthtest1", tp, 200); + + String tableName = "cadcauthtest1.testCreateUniqueIndex"; + TableDesc td = doCreateTable(schemaOwner, tableName); + for (ColumnDesc cd : td.getColumnDescs()) { + + ExecutionPhase expected = ExecutionPhase.COMPLETED; + if (cd.getColumnName().startsWith("e") || cd.getColumnName().startsWith("a")) { + expected = ExecutionPhase.ERROR; // unique index not allowed + } + log.info("testCreateUniqueIndex: " + cd.getColumnName() + " expect: " + expected.getValue()); + doCreateIndex(schemaOwner, tableName, cd.getColumnName(), true, expected, null); + } + + // cleanup on success + doDelete(schemaOwner, tableName, false); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + private void compare(TableDesc expected, TableDesc actual) { + // When you read just a single table document you do not get the schema name and TableReader makes one up + //Assert.assertEquals("schema name", "default", actual.getSchemaName()); + + Assert.assertEquals("table name", expected.getTableName(), actual.getTableName()); + Assert.assertEquals("table description", expected.description, actual.description); + + Assert.assertEquals("num columns", expected.getColumnDescs().size(), actual.getColumnDescs().size()); + for (int i = 0; i < expected.getColumnDescs().size(); i++) { + ColumnDesc ecd = expected.getColumnDescs().get(i); + ColumnDesc acd = actual.getColumnDescs().get(i); + Assert.assertEquals("column:table name", ecd.getTableName(), acd.getTableName()); + Assert.assertEquals("column name", ecd.getColumnName(), acd.getColumnName()); + Assert.assertEquals("column datatype", ecd.getDatatype(), acd.getDatatype()); + Assert.assertEquals("column description", ecd.description, acd.description); + } + } + + private void compare(VOTableTable expected, VOTableTable actual) { + // no table name or table description + for (int i = 0; i < expected.getFields().size(); i++) { + VOTableField ef = expected.getFields().get(i); + VOTableField af = actual.getFields().get(i); + Assert.assertEquals("column name", ef.getName(), af.getName()); + Assert.assertEquals("column datatype", ef.getDatatype(), af.getDatatype()); + Assert.assertEquals("column arraysize", ef.getArraysize(), af.getArraysize()); + Assert.assertEquals("column xtype", ef.xtype, af.xtype); + Assert.assertEquals("column description", ef.description, af.description); + } + } +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/LoadTableDataTest.java b/youcat/src/intTest/java/org/opencadc/youcat/LoadTableDataTest.java new file mode 100644 index 00000000..3f34571d --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/LoadTableDataTest.java @@ -0,0 +1,710 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2018. (c) 2018. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.youcat; + +import ca.nrc.cadc.dali.Circle; +import ca.nrc.cadc.dali.DoubleInterval; +import ca.nrc.cadc.dali.Point; +import ca.nrc.cadc.dali.Polygon; +import ca.nrc.cadc.dali.tables.votable.VOTableDocument; +import ca.nrc.cadc.dali.tables.votable.VOTableReader; +import ca.nrc.cadc.dali.tables.votable.VOTableResource; +import ca.nrc.cadc.dali.tables.votable.VOTableTable; +import ca.nrc.cadc.net.FileContent; +import ca.nrc.cadc.net.HttpPost; +import ca.nrc.cadc.tap.schema.TapPermissions; +import ca.nrc.cadc.vosi.actions.TableContentHandler; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URL; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.PrivilegedExceptionAction; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import javax.security.auth.Subject; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; +import uk.ac.starlink.fits.FitsTableWriter; +import uk.ac.starlink.table.StarTable; +import uk.ac.starlink.table.StarTableOutput; +import uk.ac.starlink.table.StoragePolicy; +import uk.ac.starlink.util.DataSource; +import uk.ac.starlink.votable.VOTableBuilder; + +/** + * + * @author majorb + */ +public class LoadTableDataTest extends AbstractTablesTest { + private static final Logger log = Logger.getLogger(LoadTableDataTest.class); + + private static final Charset UTF8 = Charset.forName("utf-8"); + + + public LoadTableDataTest() { + super(); + } + + private String doQuery(String testTable) throws Exception { + // TAP query check (metadata and actual table exists) + String adql = "SELECT * from " + testTable; + Map params = new TreeMap(); + params.put("LANG", "ADQL"); + params.put("QUERY", adql); + String result = Subject.doAs(anon, new AuthQueryTest.SyncQueryAction(anonQueryURL, params)); + Assert.assertNotNull(result); + return result; + } + + private VOTableTable doQueryForVOT(String testTable) throws Exception { + String result = doQuery(testTable); + VOTableReader r = new VOTableReader(); + VOTableDocument doc = r.read(result); + VOTableResource vr = doc.getResourceByType("results"); + VOTableTable vt = vr.getTable(); + Assert.assertNotNull(vt); + Assert.assertNotNull(vt.getTableData()); + return vt; + } + + @Test + public void testPostNoTableName() { + try { + log.info("start"); + + String testTable = "cadcauthtest1.testPostNoTableName"; + doCreateTable(schemaOwner, testTable); + + StringBuilder data = new StringBuilder(); + data.append("c0\n"); + data.append("string\n"); + + URL postURL = new URL(certLoadURL.toString()); + final HttpPost post = new HttpPost(postURL, new FileContent(data.toString(), TableContentHandler.CONTENT_TYPE_TSV, UTF8), false); + Subject.doAs(schemaOwner, new PrivilegedExceptionAction() { + public Object run() throws Exception { + post.run(); + return null; + } + }); + Assert.assertEquals(400, post.getResponseCode()); + Assert.assertNotNull(post.getThrowable()); + + doDelete(schemaOwner, testTable, false); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testPostInvalidColumnName() { + try { + log.info("start"); + + String testTable = "cadcauthtest1.testPostInvalidColumnName"; + doCreateTable(schemaOwner, testTable); + + StringBuilder data = new StringBuilder(); + data.append("string\n"); + data.append("string\n"); + + URL postURL = new URL(certLoadURL.toString() + "/" + testTable); + final HttpPost post = new HttpPost(postURL, new FileContent(data.toString(), TableContentHandler.CONTENT_TYPE_TSV, UTF8), false); + Subject.doAs(schemaOwner, new PrivilegedExceptionAction() { + public Object run() throws Exception { + post.run(); + return null; + } + }); + Assert.assertEquals(400, post.getResponseCode()); + Assert.assertNotNull(post.getThrowable()); + + doDelete(schemaOwner, testTable, false); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testWrongNumberOfColumns() { + try { + log.info("start"); + + String testTable = "cadcauthtest1.testWrongNumberOfColumns"; + doCreateTable(schemaOwner, testTable); + + StringBuilder data = new StringBuilder(); + data.append("c0\tc1\n"); + data.append("string"); + + URL postURL = new URL(certLoadURL.toString() + "/" + testTable); + final HttpPost post = new HttpPost(postURL, new FileContent(data.toString(), TableContentHandler.CONTENT_TYPE_TSV, UTF8), false); + Subject.doAs(schemaOwner, new PrivilegedExceptionAction() { + public Object run() throws Exception { + post.run(); + return null; + } + }); + Assert.assertEquals(400, post.getResponseCode()); + Assert.assertNotNull(post.getThrowable()); + + doDelete(schemaOwner, testTable, false); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testNoSuchTable() { + try { + log.info("start"); + + StringBuilder data = new StringBuilder(); + data.append("c0\tc1\n"); + data.append("string"); + + URL postURL = new URL(certLoadURL.toString() + "/cadcauthtest1.noSuchTable"); + final HttpPost post = new HttpPost(postURL, new FileContent(data.toString(), TableContentHandler.CONTENT_TYPE_TSV, UTF8), false); + Subject.doAs(schemaOwner, new PrivilegedExceptionAction() { + public Object run() throws Exception { + post.run(); + return null; + } + }); + log.info(post.getThrowable()); + Assert.assertEquals(404, post.getResponseCode()); + Assert.assertNotNull(post.getThrowable()); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testInvalidTableName() { + try { + log.info("start"); + + StringBuilder data = new StringBuilder(); + data.append("c0\tc1\n"); + data.append("string"); + + URL postURL = new URL(certLoadURL.toString() + "/cadcauthtest1.invalid.table.name"); + final HttpPost post = new HttpPost(postURL, new FileContent(data.toString(), TableContentHandler.CONTENT_TYPE_TSV, UTF8), false); + Subject.doAs(schemaOwner, new PrivilegedExceptionAction() { + public Object run() throws Exception { + post.run(); + return null; + } + }); + Assert.assertEquals(404, post.getResponseCode()); + Assert.assertNotNull(post.getThrowable()); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testNotTableOwner() { + try { + log.info("start"); + + clearSchemaPerms(); + + String testTable = "cadcauthtest1.testNotTableOwner"; + doCreateTable(schemaOwner, testTable); + + StringBuilder data = new StringBuilder(); + data.append("c0\tc1\n"); + data.append("string"); + + URL postURL = new URL(certLoadURL.toString() + "/" + testTable); + final HttpPost post = new HttpPost(postURL, new FileContent(data.toString(), TableContentHandler.CONTENT_TYPE_TSV, UTF8), false); + Subject.doAs(subjectWithGroups, new PrivilegedExceptionAction() { + public Object run() throws Exception { + post.run(); + return null; + } + }); + Assert.assertEquals(403, post.getResponseCode()); + Assert.assertNotNull(post.getThrowable()); + + doDelete(schemaOwner, testTable, false); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testAllDataTypesTSV() { + try { + log.info("start"); + + TapPermissions tp = new TapPermissions(null, true, null, null); + setPerms(schemaOwner, "cadcauthtest1", tp, 200); + + String testTable = "cadcauthtest1.testAllDataTypesTSV"; + doCreateTable(schemaOwner, testTable); + setPerms(schemaOwner, testTable, tp, 200); + + StringBuilder data = new StringBuilder(); + data.append("c0\tc1\tc2\tc3\tc4\tc5\tc6\te7\te8\te9\te10\n"); + for (int i = 0; i < 10; i++) { + data.append("string" + i).append("\t"); + data.append(Short.MAX_VALUE).append("\t"); + data.append(Integer.MAX_VALUE).append("\t"); + data.append(Long.MAX_VALUE).append("\t"); + data.append(Float.MAX_VALUE).append("\t"); + data.append(Double.MAX_VALUE).append("\t"); + data.append("2018-11-05T22:12:33.111").append("\t"); + data.append("1.0 2.0").append("\t"); // interval + data.append("1.0 2.0").append("\t"); // point + data.append("1.0 2.0 3.0").append("\t"); // circle + data.append("1.0 2.0 3.0 4.0 5.0 6.0").append("\n"); // polygon + } + + URL postURL = new URL(certLoadURL.toString() + "/" + testTable); + final HttpPost post = new HttpPost(postURL, new FileContent(data.toString(), TableContentHandler.CONTENT_TYPE_TSV, UTF8), false); + Subject.doAs(schemaOwner, new PrivilegedExceptionAction() { + public Object run() throws Exception { + post.run(); + return null; + } + }); + Assert.assertNull(post.getThrowable()); + Assert.assertEquals(200, post.getResponseCode()); + + VOTableTable vt = doQueryForVOT(testTable); + Iterator> it = vt.getTableData().iterator(); + int count = 0; + while (it.hasNext()) { + List next = it.next(); + Assert.assertEquals("string" + count, (String) next.get(0)); + Assert.assertEquals(new Short(Short.MAX_VALUE), (Short) next.get(1)); + Assert.assertEquals(new Integer(Integer.MAX_VALUE), (Integer) next.get(2)); + Assert.assertEquals(new Long(Long.MAX_VALUE), (Long) next.get(3)); + Assert.assertEquals(new Float(Float.MAX_VALUE), (Float) next.get(4)); + Assert.assertEquals(new Double(Double.MAX_VALUE), (Double) next.get(5)); + Assert.assertTrue(next.get(6) instanceof Date); + assertEquals(new DoubleInterval(1.0, 2.0), (DoubleInterval) next.get(7)); + assertEquals(new Point(1.0, 2.0), (Point) next.get(8)); + assertEquals(new Circle(new Point(1, 2), 3), (Circle) next.get(9)); + Polygon p = new Polygon(); + p.getVertices().add(new Point(1.0, 2.0)); + p.getVertices().add(new Point(3.0, 4.0)); + p.getVertices().add(new Point(5.0, 6.0)); + assertEquals(p, (Polygon) next.get(10)); + count++; + } + Assert.assertEquals(10, count); + + doDelete(schemaOwner, testTable, false); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testAllDataTypesFITS() { + try { + log.info("start"); + + TapPermissions tp = new TapPermissions(null, true, null, null); + setPerms(schemaOwner, "cadcauthtest1", tp, 200); + + String testTable = "cadcauthtest1.testAllDataTypesFits"; + doCreateTable(schemaOwner, testTable); + setPerms(schemaOwner, testTable, tp, 200); + + StringBuilder data = new StringBuilder(); + data.append("c0\tc1\tc2\tc3\tc4\tc5\tc6\te7\te8\te9\te10\n"); + //data.append("c0\tc1\tc2\tc3\tc4\tc5\n"); + for (int i = 0; i < 10; i++) { + data.append("string" + i).append("\t"); + data.append(Short.MAX_VALUE).append("\t"); + data.append(Integer.MAX_VALUE).append("\t"); + data.append(Long.MAX_VALUE).append("\t"); + data.append(Float.MAX_VALUE).append("\t"); + data.append(Double.MAX_VALUE).append("\t"); + data.append("2018-11-05T22:12:33.111").append("\t"); + data.append("1.0 2.0").append("\t"); // interval + data.append("1.0 2.0").append("\t"); // point + data.append("1.0 2.0 3.0").append("\t"); // circle + data.append("1.0 2.0 3.0 4.0 5.0 6.0").append("\n"); // polygon + } + + URL postURL = new URL(certLoadURL.toString() + "/" + testTable); + final HttpPost post = new HttpPost(postURL, data.toString(), TableContentHandler.CONTENT_TYPE_TSV, false); + Subject.doAs(schemaOwner, new PrivilegedExceptionAction() { + public Object run() throws Exception { + post.run(); + return null; + } + }); + Assert.assertNull(post.getThrowable()); + Assert.assertEquals(200, post.getResponseCode()); + + final String voTableString = doQuery(testTable); + + log.info("VOTable table: " + voTableString); + + // convert to fits table + DataSource ds = new DataSource() { + protected InputStream getRawInputStream() throws IOException { + return new ByteArrayInputStream(voTableString.getBytes()); + } + }; + VOTableBuilder voTableBuilder = new VOTableBuilder(); + StarTable st = voTableBuilder.makeStarTable(ds, false, StoragePolicy.PREFER_MEMORY); + StarTableOutput tableOutput = new StarTableOutput(); + String fitsFilename = "build/tmp/" + testTable + ".fits"; + File fitsFile = new File(fitsFilename); + FileOutputStream out = new FileOutputStream(fitsFile); + tableOutput.writeStarTable(st, out, new FitsTableWriter()); + out.flush(); + + doDelete(schemaOwner, testTable, false); + doCreateTable(schemaOwner, testTable); + setPerms(schemaOwner, testTable, tp, 200); + + // Post the FITS table + byte[] bytes = Files.readAllBytes(Paths.get(fitsFilename)); + FileContent fileContent = new FileContent(bytes, "application/fits"); + final HttpPost post2 = new HttpPost(postURL, fileContent, false); + Subject.doAs(schemaOwner, new PrivilegedExceptionAction() { + public Object run() throws Exception { + post2.run(); + return null; + } + }); + Assert.assertNull(post2.getThrowable()); + Assert.assertEquals(200, post2.getResponseCode()); + + // next: query the table and assert results are correct + VOTableTable vt = doQueryForVOT(testTable); + Iterator> it = vt.getTableData().iterator(); + int count = 0; + while (it.hasNext()) { + List next = it.next(); + Assert.assertEquals("string" + count, (String) next.get(0)); + Assert.assertEquals(new Short(Short.MAX_VALUE), (Short) next.get(1)); + Assert.assertEquals(new Integer(Integer.MAX_VALUE), (Integer) next.get(2)); + Assert.assertEquals(new Long(Long.MAX_VALUE), (Long) next.get(3)); + Assert.assertEquals(new Float(Float.MAX_VALUE), (Float) next.get(4)); + Assert.assertEquals(new Double(Double.MAX_VALUE), (Double) next.get(5)); + Assert.assertTrue(next.get(6) instanceof Date); + assertEquals(new DoubleInterval(1.0, 2.0), (DoubleInterval) next.get(7)); + assertEquals(new Point(1.0, 2.0), (Point) next.get(8)); + assertEquals(new Circle(new Point(1, 2), 3), (Circle) next.get(9)); + Polygon p = new Polygon(); + p.getVertices().add(new Point(1.0, 2.0)); + p.getVertices().add(new Point(3.0, 4.0)); + p.getVertices().add(new Point(5.0, 6.0)); + assertEquals(p, (Polygon) next.get(10)); + count++; + } + Assert.assertEquals(10, count); + + doDelete(schemaOwner, testTable, false); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testMixedContentTypeASCII() { + try { + log.info("start"); + + TapPermissions tp = new TapPermissions(null, true, null, null); + setPerms(schemaOwner, "cadcauthtest1", tp, 200); + + String testTable = "cadcauthtest1.testMixedContentTypeASCII"; + doCreateTable(schemaOwner, testTable); + setPerms(schemaOwner, testTable, tp, 200); + + StringBuilder data = new StringBuilder(); + data.append("c0, c6, c2\n"); + for (int i=0; i<10; i++) { + data.append("string" + i + ",2018-11-05T22:12:33.111," + i + "\n"); + } + + URL postURL = new URL(certLoadURL.toString() + "/" + testTable); + final HttpPost post1 = new HttpPost(postURL, new FileContent(data.toString(), TableContentHandler.CONTENT_TYPE_CSV, UTF8), false); + Subject.doAs(schemaOwner, new PrivilegedExceptionAction() { + public Object run() throws Exception { + post1.run(); + return null; + } + }); + Assert.assertNull(post1.getThrowable()); + Assert.assertEquals(200, post1.getResponseCode()); + + VOTableTable vt = doQueryForVOT(testTable); + Iterator> it = vt.getTableData().iterator(); + int count = 0; + while (it.hasNext()) { + List next = it.next(); + Assert.assertEquals("string" + count, (String) next.get(0)); + Assert.assertTrue(next.get(6) instanceof Date); + Assert.assertEquals(count, ((Integer) next.get(2)).intValue()); + count++; + } + Assert.assertEquals(10, count); + + data = new StringBuilder(); + data.append("c0\tc6\tc2\n"); + for (int i=10; i<20; i++) { + data.append("string" + i + "\t2018-11-05T22:12:33.111\t" + i + "\n"); + } + + final HttpPost post2 = new HttpPost(postURL, new FileContent(data.toString(), TableContentHandler.CONTENT_TYPE_TSV, UTF8), false); + Subject.doAs(schemaOwner, new PrivilegedExceptionAction() { + public Object run() throws Exception { + post2.run(); + return null; + } + }); + Assert.assertNull(post2.getThrowable()); + Assert.assertEquals(200, post2.getResponseCode()); + + vt = doQueryForVOT(testTable); + it = vt.getTableData().iterator(); + count = 0; + while (it.hasNext()) { + List next = it.next(); + Assert.assertEquals("string" + count, (String) next.get(0)); + Assert.assertTrue(next.get(6) instanceof Date); + Assert.assertEquals(count, ((Integer) next.get(2)).intValue()); + count++; + } + Assert.assertEquals(20, count); + + doDelete(schemaOwner, testTable, false); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testMultipleBatches() { + try { + log.info("start"); + + TapPermissions tp = new TapPermissions(null, true, null, null); + setPerms(schemaOwner, "cadcauthtest1", tp, 200); + + String testTable = "cadcauthtest1.testMultipleBatches"; + doCreateTable(schemaOwner, testTable); + setPerms(schemaOwner, testTable, tp, 200); + + StringBuilder data = new StringBuilder(); + data.append("c0\tc6\tc2\n"); + for (int i=0; i<3500; i++) { + data.append("string" + i + "\t2018-11-05T22:12:33.111\t" + i + "\n"); + } + + URL postURL = new URL(certLoadURL.toString() + "/" + testTable); + final HttpPost post = new HttpPost(postURL, new FileContent(data.toString(), TableContentHandler.CONTENT_TYPE_TSV, UTF8), false); + Subject.doAs(schemaOwner, new PrivilegedExceptionAction() { + public Object run() throws Exception { + post.run(); + return null; + } + }); + Assert.assertNull(post.getThrowable()); + Assert.assertEquals(200, post.getResponseCode()); + + VOTableTable vt = doQueryForVOT(testTable); + Iterator> it = vt.getTableData().iterator(); + int count = 0; + while (it.hasNext()) { + List next = it.next(); + Assert.assertEquals("string" + count, (String) next.get(0)); + Assert.assertTrue(next.get(6) instanceof Date); + Assert.assertEquals(count, ((Integer) next.get(2)).intValue()); + count++; + } + Assert.assertEquals(3500, count); + + doDelete(schemaOwner, testTable, false); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testErrorInMiddle() { + try { + log.info("start"); + + TapPermissions tp = new TapPermissions(null, true, null, null); + setPerms(schemaOwner, "cadcauthtest1", tp, 200); + + String testTable = "cadcauthtest1.testErrorInMiddle"; + doCreateTable(schemaOwner, testTable); + setPerms(schemaOwner, testTable, tp, 200); + + StringBuilder data = new StringBuilder(); + data.append("c0\tc6\tc2\n"); + for (int i=0; i<1100; i++) { + data.append("string" + i + "\t2018-11-05T22:12:33.111\t" + i + "\n"); + } + // add in the middle a single row that has the 'int' column set to the letter 'a'. + data.append("string1101\t2018-11-05T22:12:33.111\ta\n"); + for (int i=1101; i<1200; i++) { + data.append("string" + i + "\t2018-11-05T22:12:33.111\t" + i + "\n"); + } + + URL postURL = new URL(certLoadURL.toString() + "/" + testTable); + final HttpPost post = new HttpPost(postURL, new FileContent(data.toString(), TableContentHandler.CONTENT_TYPE_TSV, UTF8), false); + Subject.doAs(schemaOwner, new PrivilegedExceptionAction() { + public Object run() throws Exception { + post.run(); + return null; + } + }); + Assert.assertNotNull(post.getThrowable()); + Assert.assertEquals(400, post.getResponseCode()); + log.info("response message: " + post.getResponseBody()); + + // make sure the first 1000 (batch size) got in + VOTableTable vt = doQueryForVOT(testTable); + Iterator> it = vt.getTableData().iterator(); + int count = 0; + while (it.hasNext()) { + List next = it.next(); + Assert.assertEquals("string" + count, (String) next.get(0)); + Assert.assertTrue(next.get(6) instanceof Date); + Assert.assertEquals(count, ((Integer) next.get(2)).intValue()); + count++; + } + Assert.assertEquals(1000, count); + log.info("Count: " + count); + + doDelete(schemaOwner, testTable, false); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + private void assertEquals(DoubleInterval expected, DoubleInterval actual) { + Assert.assertEquals(expected.getLower(), actual.getLower(), 1.0e-9); + Assert.assertEquals(expected.getUpper(), actual.getUpper(), 1.0e-9); + + } + + private void assertEquals(Point expected, Point actual) { + Assert.assertEquals(expected.getLongitude(), actual.getLongitude(), 1.0e-9); + Assert.assertEquals(expected.getLatitude(), actual.getLatitude(), 1.0e-9); + } + + private void assertEquals(Circle expected, Circle actual) { + assertEquals(expected.getCenter(), actual.getCenter()); + Assert.assertEquals(expected.getRadius(), actual.getRadius(), 1.0e-9); + } + + private void assertEquals(Polygon expected, Polygon actual) { + Assert.assertEquals("num vertices", expected.getVertices().size(), actual.getVertices().size()); + for (int i=0; i < expected.getVertices().size(); i++) { + Point ep = expected.getVertices().get(i); + Point ap = actual.getVertices().get(i); + assertEquals(ep, ap); + } + } +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/PermissionsTest.java b/youcat/src/intTest/java/org/opencadc/youcat/PermissionsTest.java new file mode 100644 index 00000000..922721f9 --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/PermissionsTest.java @@ -0,0 +1,913 @@ +/* +************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2019. (c) 2019. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.youcat; + +import java.io.ByteArrayOutputStream; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URL; +import java.nio.charset.Charset; +import java.security.PrivilegedExceptionAction; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +import javax.security.auth.Subject; +import javax.security.auth.x500.X500Principal; + +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; +import org.opencadc.gms.GroupURI; + +import ca.nrc.cadc.auth.HttpPrincipal; +import ca.nrc.cadc.auth.RunnableAction; +import ca.nrc.cadc.dali.tables.TableData; +import ca.nrc.cadc.dali.tables.votable.VOTableDocument; +import ca.nrc.cadc.dali.tables.votable.VOTableReader; +import ca.nrc.cadc.dali.tables.votable.VOTableResource; +import ca.nrc.cadc.dali.tables.votable.VOTableTable; +import ca.nrc.cadc.net.FileContent; +import ca.nrc.cadc.net.HttpDownload; +import ca.nrc.cadc.net.HttpPost; +import ca.nrc.cadc.tap.schema.TapPermissions; +import ca.nrc.cadc.uws.ExecutionPhase; +import ca.nrc.cadc.vosi.actions.TableContentHandler; + +/** + * + * @author majorb + */ +public class PermissionsTest extends AbstractTablesTest { + + private static final Logger log = Logger.getLogger(PermissionsTest.class); + + public PermissionsTest() { + super(); + } + + @Test + public void testAnon() { + log.info("testGetAnon()"); + try { + clearSchemaPerms(); + + String testSchema = "cadcauthtest1"; + String testTable = testSchema + ".testGetAnon"; + doCreateTable(schemaOwner, testTable); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + URL schemaPerms = new URL(permsURL.toString() + "/" + testSchema); + URL tablePerms = new URL(permsURL.toString() + "/" + testTable); + + // get schema perms + HttpDownload get = new HttpDownload(schemaPerms, out); + get.run(); + Assert.assertNotNull(get.getThrowable()); + Assert.assertEquals(get.getResponseCode(), 403); + + // get table perms + get = new HttpDownload(tablePerms, out); + get.run(); + Assert.assertNotNull(get.getThrowable()); + Assert.assertEquals(get.getResponseCode(), 403); + + String perms = + "public=true\n" + + "r-group=ivo://cadc.nrc.ca/gms?testGroup\n" + + "rw-group=ivo://cadc.nrc.ca/gms?testGroup"; + FileContent content = new FileContent(perms, "text/plain", Charset.forName("utf-8")); + + // set schema perms + HttpPost post = new HttpPost(schemaPerms, content, false); + post.run(); + Assert.assertNotNull(post.getThrowable()); + Assert.assertEquals(post.getResponseCode(), 403); + + // set table perms + post = new HttpPost(tablePerms, content, false); + post.run(); + Assert.assertNotNull(post.getThrowable()); + Assert.assertEquals(post.getResponseCode(), 403); + + doDelete(schemaOwner, testTable, false); + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testBadSetParams() { + log.info("testBadSetParams()"); + try { + clearSchemaPerms(); + + String testSchema = "cadcauthtest1"; + String testTable = testSchema + ".testBadSetParams"; + doCreateTable(schemaOwner, testTable); + + URL tablePerms = new URL(permsURL.toString() + "/" + testTable); + + // unknown parameter + FileContent content = new FileContent("junk=true", "text/plain", Charset.forName("utf-8")); + HttpPost post = new HttpPost(tablePerms, content, false); + Subject.doAs(schemaOwner, new RunnableAction(post)); + Assert.assertNotNull(post.getThrowable()); + Assert.assertEquals(post.getResponseCode(), 400); + + // bad value for boolean + content = new FileContent("public=yes", "text/plain", Charset.forName("utf-8")); + post = new HttpPost(tablePerms, content, false); + Subject.doAs(schemaOwner, new RunnableAction(post)); + Assert.assertNotNull(post.getThrowable()); + Assert.assertEquals(post.getResponseCode(), 400); + + // bad value for r-group + content = new FileContent("r-group=notagmsuri", "text/plain", Charset.forName("utf-8")); + post = new HttpPost(tablePerms, content, false); + Subject.doAs(schemaOwner, new RunnableAction(post)); + Assert.assertNotNull(post.getThrowable()); + Assert.assertEquals(post.getResponseCode(), 400); + + // bad value for rw-group + content = new FileContent("rw-group=notagmsuri", "text/plain", Charset.forName("utf-8")); + post = new HttpPost(tablePerms, content, false); + Subject.doAs(schemaOwner, new RunnableAction(post)); + Assert.assertNotNull(post.getThrowable()); + Assert.assertEquals(post.getResponseCode(), 400); + + // cannot set owner + content = new FileContent("owner=me", "text/plain", Charset.forName("utf-8")); + post = new HttpPost(tablePerms, content, false); + Subject.doAs(schemaOwner, new RunnableAction(post)); + Assert.assertNotNull(post.getThrowable()); + Assert.assertEquals(post.getResponseCode(), 400); + + doDelete(schemaOwner, testTable, false); + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testPublic() { + log.info("testPublic()"); + try { + clearSchemaPerms(); + + String testSchema = "cadcauthtest1"; + String testTable = testSchema + ".testPublic"; + doCreateTable(schemaOwner, testTable); + + this.doQuery(anon, anonQueryURL, testTable, 400); + + TapPermissions tp = new TapPermissions(null, true, null, null); + setPerms(schemaOwner, testSchema, tp, 200); + this.doQuery(anon, anonQueryURL, testTable, 403); + + setPerms(schemaOwner, testTable, tp, 200); + this.doQuery(anon, anonQueryURL, testTable, 200); + + setPerms(anon, testTable, tp, 403); + + doDelete(schemaOwner, testTable, false); + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testGroupRead() { + log.info("testGroupRead()"); + try { + + clearSchemaPerms(); + + String testSchema = "cadcauthtest1"; + String testTable = testSchema + ".testGroupRead"; + doCreateTable(schemaOwner, testTable); + + this.doQuery(subjectWithGroups, certQueryURL, testTable, 400); + this.insertData(subjectWithGroups, certLoadURL, testTable, 403); + this.doCreateIndex(subjectWithGroups, testTable, "c0", false, ExecutionPhase.ERROR, "permission denied"); + + GroupURI readGroup = new GroupURI(VALID_TEST_GROUP); + TapPermissions tp = new TapPermissions(null, false, readGroup, null); + setPerms(schemaOwner, testSchema, tp, 200); + TapPermissions tp1 = getPermissions(schemaOwner, testSchema, 200); + Assert.assertNotNull(tp1.isPublic); + Assert.assertFalse(tp1.isPublic); + Assert.assertEquals(readGroup, tp1.readGroup); + Assert.assertNull(tp1.readWriteGroup); + + this.doQuery(subjectWithGroups, certQueryURL, testTable, 403); + this.insertData(subjectWithGroups, certLoadURL, testTable, 403); + this.doCreateIndex(subjectWithGroups, testTable, "c0", false, ExecutionPhase.ERROR, "permission denied"); + + setPerms(schemaOwner, testTable, tp, 200); + tp1 = getPermissions(schemaOwner, testTable, 200); + Assert.assertNotNull(tp1.isPublic); + Assert.assertFalse(tp1.isPublic); + Assert.assertEquals(readGroup, tp1.readGroup); + Assert.assertNull(tp1.readWriteGroup); + + this.doQuery(subjectWithGroups, certQueryURL, testTable, 200); + this.insertData(subjectWithGroups, certLoadURL, testTable, 403); + this.doCreateIndex(subjectWithGroups, testTable, "c0", false, ExecutionPhase.ERROR, "permission denied"); + + doDelete(schemaOwner, testTable, false); + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testGroupReadWrite() { + log.info("testGroupReadWrite()"); + try { + clearSchemaPerms(); + + String testSchema = "cadcauthtest1"; + String testTable = testSchema + ".testGroupReadWrite"; + doCreateTable(schemaOwner, testTable); + + this.doQuery(subjectWithGroups, certQueryURL, testTable, 400); + this.insertData(subjectWithGroups, certLoadURL, testTable, 403); + this.doCreateIndex(subjectWithGroups, testTable, "c0", false, ExecutionPhase.ERROR, "permission denied");; + + GroupURI readWriteGroup = new GroupURI(VALID_TEST_GROUP); + TapPermissions tp = new TapPermissions(null, false, null, readWriteGroup); + setPerms(schemaOwner, testSchema, tp, 200); + TapPermissions tp1 = getPermissions(schemaOwner, testSchema, 200); + Assert.assertNotNull(tp1.isPublic); + Assert.assertFalse(tp1.isPublic); + Assert.assertNull(tp1.readGroup); + Assert.assertEquals(readWriteGroup, tp1.readWriteGroup); + + this.doQuery(subjectWithGroups, certQueryURL, testTable, 403); + this.insertData(subjectWithGroups, certLoadURL, testTable, 403); + this.doCreateIndex(subjectWithGroups, testTable, "c0", false, ExecutionPhase.ERROR, "permission denied"); + + setPerms(schemaOwner, testTable, tp, 200); + tp1 = getPermissions(schemaOwner, testTable, 200); + Assert.assertNotNull(tp1.isPublic); + Assert.assertFalse(tp1.isPublic); + Assert.assertNull(tp1.readGroup); + Assert.assertEquals(readWriteGroup, tp1.readWriteGroup); + + this.doQuery(subjectWithGroups, certQueryURL, testTable, 200); + this.insertData(subjectWithGroups, certLoadURL, testTable, 200); + this.doCreateIndex(subjectWithGroups, testTable, "c0", false, ExecutionPhase.COMPLETED, null); + + doDelete(schemaOwner, testTable, false); + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testSchemaOwnerDropTable() { + log.info("testDropTable()"); + try { + clearSchemaPerms(); + + String testSchema = "cadcauthtest1"; + String testTable = testSchema + ".testDropTable"; + + TapPermissions tp = new TapPermissions(null, true, null, new GroupURI(VALID_TEST_GROUP)); + setPerms(schemaOwner, testSchema, tp, 200); + + doCreateTable(subjectWithGroups, testTable); + this.doQuery(subjectWithGroups, certQueryURL, testTable, 200); + + doDelete(schemaOwner, testTable, true); + + // query should fail with bad request now + this.doQuery(subjectWithGroups, certQueryURL, testTable, 400); + + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testDropTable() { + log.info("testDropTable()"); + try { + clearSchemaPerms(); + + String testSchema = "cadcauthtest1"; + String testTable = testSchema + ".testDropTable"; + + TapPermissions tp = new TapPermissions(null, true, null, new GroupURI(VALID_TEST_GROUP)); + setPerms(schemaOwner, testSchema, tp, 200); + + doCreateTable(subjectWithGroups, testTable); + this.doQuery(subjectWithGroups, certQueryURL, testTable, 200); + this.setPerms(subjectWithGroups, testTable, tp, 200); + + doDelete(subjectWithGroups, testTable, true); + + // query should fail now + this.doQuery(subjectWithGroups, certQueryURL, testTable, 400); + + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testNoInheritance() { + log.info("testNoInheritance()"); + try { + + String testSchema = "cadcauthtest1"; + + GroupURI group1 = new GroupURI("ivo://cadc.nrc.ca/gms?group1"); + GroupURI group2 = new GroupURI("ivo://cadc.nrc.ca/gms?group2"); + TapPermissions tp = new TapPermissions(null, true, group1, group2); + this.setPerms(schemaOwner, testSchema, tp, 200); + + TapPermissions actual = this.getPermissions(schemaOwner, testSchema, 200); + Assert.assertTrue(actual.owner.getPrincipals(X500Principal.class).iterator().next() + .getName().equals("CN=cadcauthtest1_24c,OU=cadc,O=hia,C=ca")); + Assert.assertEquals(true, actual.isPublic); + Assert.assertEquals(group1, actual.readGroup); + Assert.assertEquals(group2, actual.readWriteGroup); + + String testTable = testSchema + ".testNoInheritance"; + doCreateTable(schemaOwner, testTable); + + actual = this.getPermissions(schemaOwner, testTable, 200); + Assert.assertTrue(actual.owner.getPrincipals(X500Principal.class).iterator().next() + .getName().equals("CN=cadcauthtest1_24c,OU=cadc,O=hia,C=ca")); + Assert.assertEquals(false, actual.isPublic); + Assert.assertNull(actual.readGroup); + Assert.assertNull(actual.readWriteGroup); + + doDelete(schemaOwner, testTable, false); + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testQueriesChangingPerms() { + log.info("testQueriesChangingPerms()"); + try { + + clearSchemaPerms(); + + // query tap_schema.schemas -- null owner so should be public + this.doQuery(anon, anonQueryURL, "tap_schema.schemas", 200); + + String testSchema = "cadcauthtest1"; + String testTable = testSchema + ".testQueriesChangingPerms"; + doCreateTable(schemaOwner, testTable); + + // initially private + this.doQuery(anon, certQueryURL, testTable, 400); + this.doQuery(subjectWithGroups, certQueryURL, testTable, 400); + this.doQuery(schemaOwner, certQueryURL, testTable, 200); + + // set schema and table to public + TapPermissions tp = new TapPermissions(null, true, null, null); + this.setPerms(schemaOwner, testSchema, tp, 200); + this.setPerms(schemaOwner, testTable, tp, 200); + this.doQuery(anon, certQueryURL, testTable, 200); + this.doQuery(subjectWithGroups, certQueryURL, testTable, 200); + this.doQuery(schemaOwner, certQueryURL, testTable, 200); + + // remove public from table + tp = new TapPermissions(null, false, null, null); + this.setPerms(schemaOwner, testTable, tp, 200); + this.doQuery(anon, certQueryURL, testTable, 403); + this.doQuery(subjectWithGroups, certQueryURL, testTable, 403); + this.doQuery(schemaOwner, certQueryURL, testTable, 200); + + // add group read + tp = new TapPermissions(null, false, new GroupURI(VALID_TEST_GROUP), null); + this.setPerms(schemaOwner, testTable, tp, 200); + this.doQuery(anon, certQueryURL, testTable, 403); + this.doQuery(subjectWithGroups, certQueryURL, testTable, 200); + this.doQuery(schemaOwner, certQueryURL, testTable, 200); + + // remove group read, add group read-write + tp = new TapPermissions(null, false, null, new GroupURI(VALID_TEST_GROUP)); + this.setPerms(schemaOwner, testTable, tp, 200); + this.doQuery(anon, certQueryURL, testTable, 403); + this.doQuery(subjectWithGroups, certQueryURL, testTable, 200); + this.doQuery(schemaOwner, certQueryURL, testTable, 200); + + //doDelete(schemaOwner, testTable, false); + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testAnonQuerySchemasTable() { + log.info("testAnonQuerySchemasTable()"); + try { + clearSchemaPerms(); + + String query = "select schema_name from tap_schema.schemas"; + + VOTableDocument doc = doQueryWithResults(anon, anonQueryURL, query); + + assertAnonymousSchemaResults(doc); + + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testOwnerQuerySchemasTable() { + log.info("testOwnerQuerySchemasTable()"); + try { + clearSchemaPerms(); + + String query = "select schema_name from tap_schema.schemas"; + + VOTableDocument doc = doQueryWithResults(schemaOwner, certQueryURL, query); + + assertAuthtest1ReadResults(doc); + + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testGroupAccessQuerySchemasTable() { + log.info("testGroupAccessQuerySchemasTable()"); + try { + clearSchemaPerms(); + + GroupURI readGroup = new GroupURI(VALID_TEST_GROUP); + TapPermissions tp = new TapPermissions(null, false, readGroup, null); + this.setPerms(this.schemaOwner, "cadcauthtest1", tp, 200); + + String query = "select schema_name from tap_schema.schemas"; + + VOTableDocument doc = doQueryWithResults(subjectWithGroups, certQueryURL, query); + + assertAuthtest1ReadResults(doc); + + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testAnonQueryTablesTable() { + log.info("testAnonQueryTablesTable()"); + try { + clearSchemaPerms(); + + String query = "select schema_name from tap_schema.tables"; + VOTableDocument doc = doQueryWithResults(anon, anonQueryURL, query); + + assertAnonymousSchemaResults(doc); + + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testOwnerQueryTablesTable() { + log.info("testOwnerQueryTablesTable()"); + try { + clearSchemaPerms(); + + String query = "select schema_name from tap_schema.tables"; + + VOTableDocument doc = doQueryWithResults(schemaOwner, certQueryURL, query); + + assertAuthtest1ReadResults(doc); + + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testGroupAccessQueryTablesTable() { + log.info("testGroupAccessQueryTablesTable()"); + try { + clearSchemaPerms(); + + GroupURI readGroup = new GroupURI(VALID_TEST_GROUP); + TapPermissions tp = new TapPermissions(null, false, readGroup, null); + this.setPerms(this.schemaOwner, "cadcauthtest1", tp, 200); + + String query = "select schema_name from tap_schema.tables"; + + VOTableDocument doc = doQueryWithResults(subjectWithGroups, certQueryURL, query); + + assertAuthtest1ReadResults(doc); + + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testAnonQueryColumnsTable() { + log.info("testAnonQueryColumnsTable()"); + try { + clearSchemaPerms(); + + String query = "select t.schema_name from tap_schema.tables t " + + "join tap_schema.columns c on t.table_name=c.table_name"; + VOTableDocument doc = doQueryWithResults(anon, anonQueryURL, query); + + assertAnonymousSchemaResults(doc); + + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + + } + + @Test + public void testOwnerQueryColumnsTable() { + log.info("testOwnerQueryColumnsTable()"); + try { + clearSchemaPerms(); + + String query = "select t.schema_name from tap_schema.tables t " + + "join tap_schema.columns c on t.table_name=c.table_name"; + + VOTableDocument doc = doQueryWithResults(schemaOwner, certQueryURL, query); + + assertAuthtest1ReadResults(doc); + + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + @Test + public void testGroupQueryColumnsTable() { + log.info("testGroupQueryColumnsTable()"); + try { + clearSchemaPerms(); + + GroupURI readGroup = new GroupURI(VALID_TEST_GROUP); + TapPermissions tp = new TapPermissions(null, false, readGroup, null); + this.setPerms(this.schemaOwner, "cadcauthtest1", tp, 200); + + String query = "select t.schema_name from tap_schema.tables t " + + "join tap_schema.columns c on t.table_name=c.table_name"; + + VOTableDocument doc = doQueryWithResults(subjectWithGroups, certQueryURL, query); + + assertAuthtest1ReadResults(doc); + + } catch (Exception t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + private void assertAnonymousSchemaResults(VOTableDocument doc) { + + try { + + VOTableResource results = doc.getResourceByType("results"); + VOTableTable table = results.getTable(); + TableData data = table.getTableData(); + Iterator> rows = data.iterator(); + if (!rows.hasNext()) { + Assert.fail("no schema rows returned"); + } + + // expect at least: + // - the tap_schema schema to be present + // - the cadcauthtest1 schema to be not present + boolean foundTapSchemaSchema = false; + boolean foundCadcauthtest1Schema = false; + while (rows.hasNext()) { + List row = rows.next(); + if (((String) row.get(0)).equals("tap_schema")) { + foundTapSchemaSchema = true; + } + if (((String) row.get(0)).equals("cadcauthtest1")) { + foundCadcauthtest1Schema = true; + } + } + if (!foundTapSchemaSchema) { + Assert.fail("failed to find tap schema schema"); + } + if (foundCadcauthtest1Schema) { + Assert.fail("mistakenly found cadcauthtest1 schema"); + } + + } catch (Throwable t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + private void assertAuthtest1ReadResults(VOTableDocument doc) { + + try { + VOTableResource results = doc.getResourceByType("results"); + VOTableTable table = results.getTable(); + TableData data = table.getTableData(); + Iterator> rows = data.iterator(); + if (!rows.hasNext()) { + Assert.fail("no schema rows returned"); + } + + // expect at least: + // - the tap_schema schema to be present + // - the cadcauthtest1 schema to be present + boolean foundTapSchemaSchema = false; + boolean foundCadcauthtest1Schema = false; + while (rows.hasNext()) { + List row = rows.next(); + if (((String) row.get(0)).equals("tap_schema")) { + foundTapSchemaSchema = true; + } + if (((String) row.get(0)).equals("cadcauthtest1")) { + foundCadcauthtest1Schema = true; + } + } + if (!foundTapSchemaSchema) { + Assert.fail("failed to find tap schema schema"); + } + if (!foundCadcauthtest1Schema) { + Assert.fail("failed to find cadcauthtest1 schema"); + } + } catch (Throwable t) { + log.error("unexpected", t); + Assert.fail("unexpected: " + t.getMessage()); + } + } + + private TapPermissions getPermissions(Subject subject, String name, int expectedCode) throws MalformedURLException { + + URL getPermsURL = new URL(permsURL.toString() + "/" + name); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpDownload get = new HttpDownload(getPermsURL, out); + Subject.doAs(subject, new RunnableAction(get)); + Assert.assertEquals(get.getResponseCode(), expectedCode); + if (get.getResponseCode() == 200) { + String response = out.toString(); + log.info("get perms response: " + response); + String[] lines = response.split("\n"); + String ownerString = null; + Boolean isPublic = null; + GroupURI readGroup = null; + GroupURI readWriteGroup = null; + for (String next : lines) { + String[] parts = next.split("[=]"); + boolean hasValue = parts.length >= 2; + String key = parts[0]; + String value = next.substring(key.length() + 1); + if (key.equals("owner")) { + if (hasValue) { + ownerString = value; + } + } + if (key.equals("public")) { + if (hasValue) { + isPublic = Boolean.parseBoolean(value); + } else { + throw new RuntimeException("cannot have null public value"); + } + } + if (key.equals("r-group")) { + if (hasValue) { + readGroup = new GroupURI(URI.create(value)); + } + } + if (key.equals("rw-group")) { + if (hasValue) { + readWriteGroup = new GroupURI(URI.create(value)); + } + } + } + + Subject owner = new Subject(); + if (ownerString != null) { + X500Principal p = new X500Principal(ownerString); + owner.getPrincipals().add(p); + } + return new TapPermissions(owner, isPublic, readGroup, readWriteGroup); + } + return null; + } + + private Integer doQuery(Subject subject, URL url, String testTable, Integer expectedCode) throws Exception { + String adql = "SELECT * from " + testTable; + return doQuery(subject, url, adql, testTable, expectedCode); + } + + private Integer doQuery(Subject subject, URL url, String adql, String testTable, Integer expectedCode) throws Exception { + Map params = new TreeMap(); + params.put("LANG", "ADQL"); + params.put("QUERY", adql); + params.put("RESPONSEFORMAT", "CSV"); + Integer respCode = null; + log.info("Performing query on URL: " + url); + SyncQueryAction query = new SyncQueryAction(url, params); + if (subject != null) { + respCode = Subject.doAs(subject, query); + } else { + respCode = query.run(); + } + log.info("Query response code: " + respCode); + Assert.assertEquals(expectedCode, respCode); + return respCode; + } + + class SyncQueryAction implements PrivilegedExceptionAction { + + private URL url; + private Map params; + + public SyncQueryAction(URL url, Map params) { + this.url = url; + this.params = params; + } + + @Override + public Integer run() + throws Exception { + HttpPost doit = new HttpPost(url, params, true); + doit.run(); + + if (doit.getThrowable() != null) { + log.info("Throwable on sync query: " + doit.getThrowable()); + } + + int code = doit.getResponseCode(); + return code; + } + } + + private VOTableDocument doQueryWithResults(Subject subject, URL url, String adql) throws Exception { + Map params = new TreeMap(); + params.put("LANG", "ADQL"); + params.put("QUERY", adql); + VOTableDocument ret = null; + log.info("Performing query on URL: " + url); + SyncQueryActionWithResults query = new SyncQueryActionWithResults(url, params); + if (subject != null) { + ret = Subject.doAs(subject, query); + } else { + ret = query.run(); + } + return ret; + } + + class SyncQueryActionWithResults implements PrivilegedExceptionAction { + + private URL url; + private Map params; + + public SyncQueryActionWithResults(URL url, Map params) { + this.url = url; + this.params = params; + } + + @Override + public VOTableDocument run() + throws Exception { + HttpPost doit = new HttpPost(url, params, true); + doit.prepare(); + + if (doit.getThrowable() != null) { + log.info("Throwable on sync query: " + doit.getThrowable()); + } + + int code = doit.getResponseCode(); + if (code != 200) { + throw new RuntimeException(Integer.toString(code)); + } + + VOTableReader reader = new VOTableReader(); + //String xml = doit.getResponseBody(); + //log.info("xml: " + xml); + VOTableDocument doc = reader.read(doit.getInputStream()); + + return doc; + } + } + private void insertData(Subject subject, URL url, String testTable, int expectedCode) throws Exception { + StringBuilder data = new StringBuilder(); + data.append("c0\tc1\tc2\tc3\tc4\tc5\tc6\te7\te8\te9\te10\n"); + for (int i = 0; i < 10; i++) { + data.append("string" + i).append("\t"); + data.append(Short.MAX_VALUE).append("\t"); + data.append(Integer.MAX_VALUE).append("\t"); + data.append(Long.MAX_VALUE).append("\t"); + data.append(Float.MAX_VALUE).append("\t"); + data.append(Double.MAX_VALUE).append("\t"); + data.append("2018-11-05T22:12:33.111").append("\t"); + data.append("1.0 2.0").append("\t"); // interval + data.append("1.0 2.0").append("\t"); // point + data.append("1.0 2.0 3.0").append("\t"); // circle + data.append("1.0 2.0 3.0 4.0 5.0 6.0").append("\n"); // polygon + } + insertData(subject, url, data.toString(), testTable, expectedCode); + } + + private void insertData(Subject subject, URL url, String data, String testTable, int expectedCode) throws Exception { + URL postURL = new URL(url.toString() + "/" + testTable); + log.info("Posting table data to: " + postURL); + FileContent content = new FileContent(data.toString(), TableContentHandler.CONTENT_TYPE_TSV, Charset.forName("utf-8")); + final HttpPost post = new HttpPost(postURL, content, false); + Subject.doAs(subject, new RunnableAction(post)); + Assert.assertEquals(expectedCode, post.getResponseCode()); + } + +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/RegistryClientLookupTest.java b/youcat/src/intTest/java/org/opencadc/youcat/RegistryClientLookupTest.java new file mode 100644 index 00000000..b6dc1a9f --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/RegistryClientLookupTest.java @@ -0,0 +1,363 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2019. (c) 2019. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +* $Revision: 5 $ +* +************************************************************************ + */ + +package org.opencadc.youcat; + +import ca.nrc.cadc.auth.AuthMethod; +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.auth.RunnableAction; +import ca.nrc.cadc.auth.SSLUtil; +import ca.nrc.cadc.net.HttpDownload; +import ca.nrc.cadc.net.HttpPost; +import ca.nrc.cadc.net.ResourceNotFoundException; +import ca.nrc.cadc.reg.Standards; +import ca.nrc.cadc.reg.client.RegistryClient; +import ca.nrc.cadc.util.FileUtil; +import ca.nrc.cadc.util.Log4jInit; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; +import java.util.TreeMap; + +import javax.security.auth.Subject; + +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.xerces.impl.dv.util.Base64; +import org.junit.Assert; +import org.junit.Test; +import org.opencadc.tap.TapClient; + +/** + * Half-decent test that authenticated queries work. + * + * @author pdowler + */ +public class RegistryClientLookupTest { + + private static final Logger log = Logger.getLogger(RegistryClientLookupTest.class); + + static final Subject subject; + static final String basicAuth; + static final String userid = "cadcregtest1"; + + static { + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + + File cf = FileUtil.getFileFromResource("x509_CADCRegtest1.pem", RegistryClientLookupTest.class); + subject = SSLUtil.createSubject(cf); + try { + Path passPath = Paths.get(System.getenv("A") + "/etc/" + userid + ".pass"); + byte[] password = Files.readAllBytes(passPath); + String useridPassword = new String(userid + ":" + new String(password)); + basicAuth = "Basic " + Base64.encode(useridPassword.getBytes()); + } catch (Throwable e) { + log.error("Failed to read password file", e); + throw new ExceptionInInitializerError("Failed to read password file"); + } + } + + RegistryClient regClient; + TapClient tapClient; + + Map queryParams = new TreeMap<>(); + + public RegistryClientLookupTest() throws ResourceNotFoundException { + queryParams.put("RUNID", "RegistryClientLookupTest"); + regClient = new RegistryClient(); + tapClient = new TapClient(Constants.RESOURCE_ID); + } + + @Test + public void testAnonBase() { + try { + URL url = tapClient.getAsyncURL(Standards.SECURITY_METHOD_ANON); + Assert.assertNotNull(url); + + HttpPost post = new HttpPost(url, queryParams, false); + post.run(); + Assert.assertNull(post.getThrowable()); + Assert.assertEquals(303, post.getResponseCode()); + Assert.assertNotNull(post.getRedirectURL()); + Assert.assertNull(post.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testAnonAsync() { + try { + URL url = tapClient.getAsyncURL(Standards.SECURITY_METHOD_ANON); + Assert.assertNotNull(url); + + HttpPost post = new HttpPost(url, queryParams, false); + post.run(); + Assert.assertNull(post.getThrowable()); + Assert.assertEquals(303, post.getResponseCode()); + Assert.assertNotNull(post.getRedirectURL()); + Assert.assertNull(post.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testAnonSync() { + try { + URL url = tapClient.getSyncURL(Standards.SECURITY_METHOD_ANON); + Assert.assertNotNull(url); + HttpPost post = new HttpPost(url, queryParams, false); + post.run(); + Assert.assertNull(post.getThrowable()); + Assert.assertEquals(303, post.getResponseCode()); + Assert.assertNotNull(post.getRedirectURL()); + Assert.assertNull(post.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testAnonTables() { + try { + URL url = regClient.getServiceURL(Constants.RESOURCE_ID, Standards.VOSI_TABLES_11, AuthMethod.ANON); + Assert.assertNotNull(url); + + url = new URL(url.toExternalForm() + "?detail=min"); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpDownload get = new HttpDownload(url, bos); + get.run(); + Assert.assertNull(get.getThrowable()); + Assert.assertEquals(200, get.getResponseCode()); + Assert.assertTrue(bos.size() > 0); + Assert.assertNull(get.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testX509Async() { + try { + URL url = tapClient.getAsyncURL(Standards.SECURITY_METHOD_ANON); + Assert.assertNotNull(url); + + HttpPost post = new HttpPost(url, queryParams, false); + Subject.doAs(subject, new RunnableAction(post)); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals(303, post.getResponseCode()); + Assert.assertNotNull(post.getRedirectURL()); + Assert.assertEquals(userid, post.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); + + // TODO: get job and check ownerID + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testX509Sync() { + try { + URL url = tapClient.getSyncURL(Standards.SECURITY_METHOD_CERT); + Assert.assertNotNull(url); + + HttpPost post = new HttpPost(url, queryParams, false); + Subject.doAs(subject, new RunnableAction(post)); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals(303, post.getResponseCode()); + Assert.assertNotNull(post.getRedirectURL()); + Assert.assertEquals(userid, post.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); + + // TODO: get job and check ownerID + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testX509Tables() { + try { + URL url = regClient.getServiceURL(Constants.RESOURCE_ID, Standards.VOSI_TABLES_11, AuthMethod.CERT); + Assert.assertNotNull(url); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpDownload get = new HttpDownload(url, bos); + Subject.doAs(subject, new RunnableAction(get)); + + Assert.assertNull(get.getThrowable()); + Assert.assertEquals(200, get.getResponseCode()); + Assert.assertTrue(bos.size() > 0); + Assert.assertEquals(userid, get.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + // username-password endpoints exist by convention but are not described in the + // VOSI-capabilities + @Test + public void testAuthAsync() { + try { + URL url = tapClient.getAsyncURL(Standards.SECURITY_METHOD_ANON); + url = new URL(url.toExternalForm().replace("async", "auth-async")); + + Assert.assertNotNull(url); + HttpPost post = new HttpPost(url, queryParams, false); + post.setRequestProperty(AuthenticationUtil.AUTHORIZATION_HEADER, basicAuth); + post.run(); + Assert.assertNull(post.getThrowable()); + Assert.assertEquals(303, post.getResponseCode()); + Assert.assertNotNull(post.getRedirectURL()); + Assert.assertEquals(userid, post.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testAuthSync() { + try { + URL url = tapClient.getSyncURL(Standards.SECURITY_METHOD_ANON); + url = new URL(url.toExternalForm().replace("sync", "auth-sync")); + + Assert.assertNotNull(url); + log.info("url: " + url.toString()); + HttpPost post = new HttpPost(url, queryParams, false); + post.setRequestProperty(AuthenticationUtil.AUTHORIZATION_HEADER, basicAuth); + post.run(); + Assert.assertNull(post.getThrowable()); + Assert.assertEquals(303, post.getResponseCode()); + Assert.assertNotNull(post.getRedirectURL()); + Assert.assertEquals(userid, post.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testAuthTables() { + try { + URL url = regClient.getServiceURL(Constants.RESOURCE_ID, Standards.VOSI_TABLES_11, AuthMethod.ANON); + url = new URL(url.toExternalForm().replace("tables", "auth-tables")); + + Assert.assertNotNull(url); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpDownload get = new HttpDownload(url, bos); + get.setRequestProperty(AuthenticationUtil.AUTHORIZATION_HEADER, basicAuth); + get.run(); + Assert.assertNull(get.getThrowable()); + Assert.assertEquals(200, get.getResponseCode()); + Assert.assertTrue(bos.size() > 0); + Assert.assertEquals(userid, get.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + // internal log control + @Test + public void testLogControl() { + try { + URL url = regClient.getServiceURL(Constants.RESOURCE_ID, Standards.LOGGING_CONTROL_10, AuthMethod.CERT); + Assert.assertNotNull(url); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpDownload get = new HttpDownload(url, bos); + Subject.doAs(subject, new RunnableAction(get)); + + Assert.assertNotNull(get.getThrowable()); + Assert.assertEquals(403, get.getResponseCode()); + + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/TokenAccessTest.java b/youcat/src/intTest/java/org/opencadc/youcat/TokenAccessTest.java new file mode 100644 index 00000000..c0ba7901 --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/TokenAccessTest.java @@ -0,0 +1,362 @@ +/* + ************************************************************************ + ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* + ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** + * + * (c) 2021. (c) 2021. + * Government of Canada Gouvernement du Canada + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits réservés + * + * NRC disclaims any warranties, Le CNRC dénie toute garantie + * expressed, implied, or énoncée, implicite ou légale, + * statutory, of any kind with de quelque nature que ce + * respect to the software, soit, concernant le logiciel, + * including without limitation y compris sans restriction + * any warranty of merchantability toute garantie de valeur + * or fitness for a particular marchande ou de pertinence + * purpose. NRC shall not be pour un usage particulier. + * liable in any event for any Le CNRC ne pourra en aucun cas + * damages, whether direct or être tenu responsable de tout + * indirect, special or general, dommage, direct ou indirect, + * consequential or incidental, particulier ou général, + * arising from the use of the accessoire ou fortuit, résultant + * software. Neither the name de l'utilisation du logiciel. Ni + * of the National Research le nom du Conseil National de + * Council of Canada nor the Recherches du Canada ni les noms + * names of its contributors may de ses participants ne peuvent + * be used to endorse or promote être utilisés pour approuver ou + * products derived from this promouvoir les produits dérivés + * software without specific prior de ce logiciel sans autorisation + * written permission. préalable et particulière + * par écrit. + * + * This file is part of the Ce fichier fait partie du projet + * OpenCADC project. OpenCADC. + * + * OpenCADC is free software: OpenCADC est un logiciel libre ; + * you can redistribute it and/or vous pouvez le redistribuer ou le + * modify it under the terms of modifier suivant les termes de + * the GNU Affero General Public la “GNU Affero General Public + * License as published by the License” telle que publiée + * Free Software Foundation, par la Free Software Foundation + * either version 3 of the : soit la version 3 de cette + * License, or (at your option) licence, soit (à votre gré) + * any later version. toute version ultérieure. + * + * OpenCADC is distributed in the OpenCADC est distribué + * hope that it will be useful, dans l’espoir qu’il vous + * but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE + * without even the implied GARANTIE : sans même la garantie + * warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ + * or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF + * PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence + * General Public License for Générale Publique GNU Affero + * more details. pour plus de détails. + * + * You should have received Vous devriez avoir reçu une + * a copy of the GNU Affero copie de la Licence Générale + * General Public License along Publique GNU Affero avec + * with OpenCADC. If not, see OpenCADC ; si ce n’est + * . pas le cas, consultez : + * . + * + ************************************************************************ + */ + +package org.opencadc.youcat; + +import ca.nrc.cadc.auth.AuthMethod; +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.auth.AuthorizationToken; +import ca.nrc.cadc.auth.SSLUtil; +import ca.nrc.cadc.net.AuthChallenge; +import ca.nrc.cadc.net.HttpGet; +import ca.nrc.cadc.net.HttpPost; +import ca.nrc.cadc.net.NetrcFile; +import ca.nrc.cadc.reg.Standards; +import ca.nrc.cadc.reg.client.RegistryClient; +import ca.nrc.cadc.util.FileUtil; +import ca.nrc.cadc.util.Log4jInit; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.net.PasswordAuthentication; +import java.net.URI; +import java.net.URL; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import java.util.Map; +import java.util.TreeMap; +import javax.security.auth.Subject; + +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; + +/** + * @author majorb + * + */ +public class TokenAccessTest { + + private static final Logger log = Logger.getLogger(TokenAccessTest.class); + + static Subject cadcAuthtest1Sub; + + static { + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.net", Level.INFO); + // need to read cert so we have creds to make the fake GMS call + File cf = FileUtil.getFileFromResource("x509_CADCAuthtest1.pem", AuthQueryTest.class); + cadcAuthtest1Sub = SSLUtil.createSubject(cf); + } + + @Test + public void useBearerTokenSuccess() { + try { + runTestSuccess(AuthenticationUtil.CHALLENGE_TYPE_BEARER); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void useBearerBadToken() { + try { + runTestAuthFail(AuthenticationUtil.CHALLENGE_TYPE_BEARER); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void useUnknownChallenge() { + try { + String token = getToken(cadcAuthtest1Sub); + log.info("token: " + token); + RegistryClient regClient = new RegistryClient(); + URL tablesURL = regClient.getServiceURL(Constants.RESOURCE_ID, Standards.VOSI_TABLES_11, AuthMethod.TOKEN); + log.info("tables url: " + tablesURL); + String domain = tablesURL.getHost(); + AuthorizationToken authToken = new AuthorizationToken("foo", token, Arrays.asList(domain)); + Subject s = new Subject(); + s.getPublicCredentials().add(authToken); + Subject.doAs(s, new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HttpGet httpGet = new HttpGet(tablesURL, new ByteArrayOutputStream()); + httpGet.run(); + Assert.assertEquals(401, httpGet.getResponseCode()); + Assert.assertFalse(httpGet.getResponseHeaderValues(AuthenticationUtil.AUTHENTICATE_HEADER).isEmpty()); + Assert.assertNull(httpGet.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); + return null; + } + }); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void useWrongDomain() { + try { + String token = getToken(cadcAuthtest1Sub); + log.info("token: " + token); + RegistryClient regClient = new RegistryClient(); + URL tablesURL = regClient.getServiceURL(Constants.RESOURCE_ID, Standards.VOSI_TABLES_11, AuthMethod.TOKEN); + log.info("tables url: " + tablesURL); + String domain = "incorrect.domain.org"; + AuthorizationToken authToken = new AuthorizationToken("Bearer", token, Arrays.asList(domain)); + Subject s = new Subject(); + s.getPublicCredentials().add(authToken); + Subject.doAs(s, new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HttpGet httpGet = new HttpGet(tablesURL, new ByteArrayOutputStream()); + httpGet.run(); + Assert.assertEquals(200, httpGet.getResponseCode()); + Assert.assertFalse(httpGet.getResponseHeaderValues(AuthenticationUtil.AUTHENTICATE_HEADER).isEmpty()); + Assert.assertNull(httpGet.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); + return null; + } + }); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + private String getToken(Subject s) throws PrivilegedActionException { + return Subject.doAs(s, new PrivilegedExceptionAction() { + public String run() throws Exception { + RegistryClient regClient = new RegistryClient(); + URL gmsURL = regClient.getServiceURL(Constants.GMS_RESOURSE_ID, Standards.SECURITY_METHOD_OAUTH, AuthMethod.CERT); + log.info("gms url: " + gmsURL); + URL authorizeURL = new URL(gmsURL.toString() + "?response_type=token"); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + HttpGet httpGet = new HttpGet(authorizeURL, out); + httpGet.run(); + if (httpGet.getResponseCode() != 200) { + throw new Exception("Failed to get token: " + httpGet.getThrowable()); + } + return out.toString(); + } + }); + } + + private String doLogin() throws Exception { + RegistryClient reg = new RegistryClient(); + URL capURL = reg.getServiceURL(Constants.RESOURCE_ID, Standards.VOSI_CAPABILITIES, AuthMethod.ANON); + HttpGet head = new HttpGet(capURL, false); + head.setHeadOnly(true); + head.prepare(); + + URL loginURL = null; + List authHeaders = head.getResponseHeaderValues("www-authenticate"); + // temp work-around for cadc-rest: it always injects an ivoa_x509 challenge + List modifiable = new ArrayList<>(); + modifiable.addAll(authHeaders); + modifiable.remove("ivoa_x509"); + authHeaders = modifiable; + + for (String s : authHeaders) { + log.info(s); + AuthChallenge c = new AuthChallenge(s); + log.info(c); + if ("ivoa_bearer".equals(c.getName()) && Standards.SECURITY_METHOD_PASSWORD.toASCIIString().equals(c.getParamValue("standard_id"))) { + loginURL = new URL(c.getParamValue("access_url")); + break; + } + } + + if (loginURL == null) { + throw new RuntimeException("no www-authenticate ivoa_bearer " + Standards.SECURITY_METHOD_PASSWORD.toASCIIString() + " challenge"); + } + + log.info("loginURL: " + loginURL); + NetrcFile netrc = new NetrcFile(); + PasswordAuthentication up = netrc.getCredentials(loginURL.getHost(), true); + if (up == null) { + throw new RuntimeException("no credentials in .netrc file for host " + loginURL.getHost()); + } + + Map params = new TreeMap<>(); + params.put("username", up.getUserName()); + params.put("password", up.getPassword()); + HttpPost login = new HttpPost(loginURL, params, true); + login.prepare(); + String token = login.getResponseHeader("x-vo-bearer"); + Assert.assertNotNull("successful login", token); + + return token; + } + + private void runTestSuccess(String challengeType) throws Exception { + //String token = getToken(cadcAuthtest1Sub); + String token = doLogin(); + log.info("token: " + token); + RegistryClient regClient = new RegistryClient(); + + URL tablesURL = regClient.getServiceURL(Constants.RESOURCE_ID, Standards.VOSI_TABLES_11, AuthMethod.TOKEN); + log.info("tables url: " + tablesURL); + String dom1 = tablesURL.getHost(); + + URL gmsURL = regClient.getServiceURL(Constants.GMS_RESOURSE_ID, Standards.GMS_SEARCH_10, AuthMethod.TOKEN); + String dom2 = gmsURL.getHost(); + + AuthorizationToken authToken = new AuthorizationToken(challengeType, token, Arrays.asList(dom1, dom2)); + log.info("token: " + authToken); + + Subject s = new Subject(); + s.getPublicCredentials().add(authToken); + Subject.doAs(s, new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HttpGet httpGet = new HttpGet(tablesURL, new ByteArrayOutputStream()); + httpGet.run(); + Assert.assertEquals(200, httpGet.getResponseCode()); + String voAuthenticated = httpGet.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER); + Assert.assertNotNull(voAuthenticated); + //Assert.assertEquals("cadcauthtest1", voAuthenticated); + Assert.assertNull(httpGet.getResponseHeader(AuthenticationUtil.AUTHENTICATE_HEADER)); + return null; + } + }); + } + + private void runTestAuthFail(String challengeType) throws Exception { + String token = "abcd"; + log.info("token: " + token); + RegistryClient regClient = new RegistryClient(); + URL tablesURL = regClient.getServiceURL(Constants.RESOURCE_ID, Standards.VOSI_TABLES_11, AuthMethod.TOKEN); + log.info("tables url: " + tablesURL); + String domain = tablesURL.getHost(); + AuthorizationToken authToken = new AuthorizationToken(challengeType, token, Arrays.asList(domain)); + Subject s = new Subject(); + s.getPublicCredentials().add(authToken); + Subject.doAs(s, new PrivilegedExceptionAction() { + @Override + public Object run() throws Exception { + HttpGet httpGet = new HttpGet(tablesURL, new ByteArrayOutputStream()); + httpGet.run(); + Assert.assertEquals(401, httpGet.getResponseCode()); + Assert.assertFalse(httpGet.getResponseHeaderValues(AuthenticationUtil.AUTHENTICATE_HEADER).isEmpty()); + Assert.assertNull(httpGet.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); + List wwwAuthValues = httpGet.getResponseHeaderValues(AuthenticationUtil.AUTHENTICATE_HEADER); + boolean errorMsgFound = false; + boolean ivoaBearerPasswordFound = false; + boolean ivoaBearerOpenIDFound = false; + boolean ivoaX509PasswordFound = false; + boolean bearerFound = false; + for (String authValue : wwwAuthValues) { + String authValueLower = authValue.toLowerCase(); + log.info("auth header: " + authValueLower); + if (authValueLower.startsWith(AuthenticationUtil.CHALLENGE_TYPE_IVOA_BEARER.toLowerCase())) { + if (authValueLower.contains(Standards.SECURITY_METHOD_PASSWORD.toString().toLowerCase())) { + ivoaBearerPasswordFound = true; + } + if (authValueLower.contains(Standards.SECURITY_METHOD_OPENID.toString().toLowerCase())) { + ivoaBearerOpenIDFound = true; + } + if (challengeType.toLowerCase().equals(AuthenticationUtil.CHALLENGE_TYPE_IVOA_BEARER.toLowerCase())) { + if (authValueLower.contains("error=\"") && authValueLower.contains("error_description=\"")) { + errorMsgFound = true; + } + } + } + if (authValueLower.startsWith(AuthenticationUtil.CHALLENGE_TYPE_IVOA_X509.toLowerCase())) { + if (authValueLower.contains(Standards.SECURITY_METHOD_HTTP_BASIC.toString().toLowerCase())) { + ivoaX509PasswordFound = true; + } + } + if (authValueLower.startsWith(AuthenticationUtil.CHALLENGE_TYPE_BEARER.toLowerCase())) { + bearerFound = true; + if (challengeType.toLowerCase().equals(AuthenticationUtil.CHALLENGE_TYPE_BEARER.toLowerCase())) { + if (authValueLower.contains("error=\"") && authValueLower.contains("error_description=\"")) { + errorMsgFound = true; + } + } + } + } + Assert.assertTrue(errorMsgFound); + Assert.assertTrue(ivoaBearerPasswordFound); + Assert.assertTrue(ivoaBearerOpenIDFound); + Assert.assertTrue(ivoaX509PasswordFound); + Assert.assertTrue(bearerFound); + return null; + } + }); + } + + +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/VosiAvailabilityTest.java b/youcat/src/intTest/java/org/opencadc/youcat/VosiAvailabilityTest.java new file mode 100644 index 00000000..1069b758 --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/VosiAvailabilityTest.java @@ -0,0 +1,95 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2011. (c) 2011. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +* $Revision: 5 $ +* +************************************************************************ +*/ + +package org.opencadc.youcat; + + +import ca.nrc.cadc.util.Log4jInit; +import ca.nrc.cadc.vosi.AvailabilityTest; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +public class VosiAvailabilityTest extends AvailabilityTest +{ + private static final Logger log = Logger.getLogger(VosiAvailabilityTest.class); + + static { + Log4jInit.setLevel("ca.nrc.cadc.vosi", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + } + + public VosiAvailabilityTest() + { + super(Constants.RESOURCE_ID); + } +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/VosiCapabilitiesTest.java b/youcat/src/intTest/java/org/opencadc/youcat/VosiCapabilitiesTest.java new file mode 100644 index 00000000..88a0655a --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/VosiCapabilitiesTest.java @@ -0,0 +1,127 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2019. (c) 2019. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +* $Revision: 5 $ +* +************************************************************************ + */ + +package org.opencadc.youcat; + +import ca.nrc.cadc.auth.AuthMethod; +import ca.nrc.cadc.reg.Capabilities; +import ca.nrc.cadc.reg.Capability; +import ca.nrc.cadc.reg.Interface; +import ca.nrc.cadc.reg.Standards; +import ca.nrc.cadc.util.Log4jInit; +import ca.nrc.cadc.vosi.CapabilitiesTest; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; + +/** + * + * @author pdowler + */ +public class VosiCapabilitiesTest extends CapabilitiesTest { + + private static final Logger log = Logger.getLogger(VosiCapabilitiesTest.class); + + static { + Log4jInit.setLevel("ca.nrc.cadc.vosi", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + } + + public VosiCapabilitiesTest() { + super(Constants.RESOURCE_ID); + } + + @Override + protected void validateContent(Capabilities caps) throws Exception { + super.validateContent(caps); + + // TAP-1.1 + Capability tap = caps.findCapability(Standards.TAP_10); + Interface base = tap.findInterface(AuthMethod.ANON, Standards.INTERFACE_PARAM_HTTP); + Assert.assertNotNull("base", base); + Assert.assertTrue("anon base", base.getSecurityMethods().contains(Standards.SECURITY_METHOD_ANON)); + Assert.assertTrue("cert base", base.getSecurityMethods().contains(Standards.SECURITY_METHOD_CERT)); + Assert.assertTrue("cookie base", base.getSecurityMethods().contains(Standards.SECURITY_METHOD_COOKIE)); + Capability tables = caps.findCapability(Standards.VOSI_TABLES_11); + Assert.assertNotNull("tables", tables); + Assert.assertNotNull("anon tables", tables.findInterface(Standards.SECURITY_METHOD_ANON, Standards.INTERFACE_PARAM_HTTP)); + Assert.assertNotNull("cert tables", tables.findInterface(Standards.SECURITY_METHOD_CERT, Standards.INTERFACE_PARAM_HTTP)); + Assert.assertNotNull("cookie tables", tables.findInterface(Standards.SECURITY_METHOD_COOKIE, Standards.INTERFACE_PARAM_HTTP)); + + Capability atu = caps.findCapability(Standards.PROTO_TABLE_UPDATE_ASYNC); + Assert.assertNotNull("table-update-async", atu); + Assert.assertNotNull("cert table-update-async", atu.findInterface(Standards.SECURITY_METHOD_CERT, Standards.INTERFACE_PARAM_HTTP)); + Assert.assertNotNull("cookie table-update-async", atu.findInterface(Standards.SECURITY_METHOD_COOKIE, Standards.INTERFACE_PARAM_HTTP)); + + Capability stu = caps.findCapability(Standards.PROTO_TABLE_LOAD_SYNC); + Assert.assertNotNull("table-load-sync", stu); + Assert.assertNotNull("cert table-load-sync", stu.findInterface(Standards.SECURITY_METHOD_CERT, Standards.INTERFACE_PARAM_HTTP)); + Assert.assertNotNull("cookie table-load-sync", stu.findInterface(Standards.SECURITY_METHOD_COOKIE, Standards.INTERFACE_PARAM_HTTP)); + } +} diff --git a/youcat/src/intTest/java/org/opencadc/youcat/VosiTablesTest.java b/youcat/src/intTest/java/org/opencadc/youcat/VosiTablesTest.java new file mode 100644 index 00000000..238b08d2 --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/VosiTablesTest.java @@ -0,0 +1,146 @@ + +package org.opencadc.youcat; + +import ca.nrc.cadc.auth.AuthMethod; +import ca.nrc.cadc.net.ContentType; +import ca.nrc.cadc.net.HttpDownload; +import ca.nrc.cadc.reg.Standards; +import ca.nrc.cadc.reg.client.RegistryClient; +import ca.nrc.cadc.tap.schema.SchemaDesc; +import ca.nrc.cadc.tap.schema.TableDesc; +import ca.nrc.cadc.tap.schema.TapSchema; +import ca.nrc.cadc.util.Log4jInit; +import ca.nrc.cadc.vosi.TableReader; +import ca.nrc.cadc.vosi.TableSetReader; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.net.URL; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; + +/** + * + * @author pdowler + */ +public class VosiTablesTest +{ + private static final Logger log = Logger.getLogger(VosiTablesTest.class); + + static + { + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + } + + URL tablesURL; + + public VosiTablesTest() + { + RegistryClient rc = new RegistryClient(); + this.tablesURL = rc.getServiceURL(Constants.RESOURCE_ID, Standards.VOSI_TABLES_11, AuthMethod.ANON); + } + + @Test + public void testValidateTablesetDoc() + { + try + { + TableSetReader tsr = new TableSetReader(true); + log.info("testValidateTablesetDoc: " + tablesURL.toExternalForm()); + + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpDownload get = new HttpDownload(tablesURL, bos); + get.run(); + Assert.assertEquals(200, get.getResponseCode()); + ContentType ct = new ContentType(get.getContentType()); + Assert.assertEquals("text/xml", ct.getBaseType()); + + TapSchema ts = tsr.read(new ByteArrayInputStream(bos.toByteArray())); + Assert.assertNotNull(ts); + } + catch(Exception unexpected) + { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testValidateTableDoc() + { + try + { + TableReader tr = new TableReader(true); + String s = tablesURL.toExternalForm() + "/tap_schema.tables"; + log.info("testValidateTableDoc: " + s); + + URL url = new URL(s); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpDownload get = new HttpDownload(url, bos); + get.run(); + Assert.assertEquals(200, get.getResponseCode()); + ContentType ct = new ContentType(get.getContentType()); + Assert.assertEquals("text/xml", ct.getBaseType()); + + TableDesc td = tr.read(new ByteArrayInputStream(bos.toByteArray())); + Assert.assertNotNull(td); + Assert.assertEquals("tap_schema.tables", td.getTableName()); + } + catch(Exception unexpected) + { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testTableNotFound() + { + try + { + String s = tablesURL.toExternalForm() + "/tap_schema.no_such_table"; + log.info("testTableNotFound: " + s); + + URL url = new URL(s); + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + HttpDownload get = new HttpDownload(url, bos); + get.run(); + Assert.assertEquals(404, get.getResponseCode()); + } + catch(Exception unexpected) + { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } + + @Test + public void testDetailMin() + { + try + { + TableSetReader tsr = new TableSetReader(true); + String s = tablesURL.toExternalForm() + "?detail=min"; + log.info("testDetailMin: " + s); + + URL url = new URL(s); + TapSchema ts = tsr.read(url.openStream()); + Assert.assertNotNull(ts); + Assert.assertFalse(ts.getSchemaDescs().isEmpty()); + SchemaDesc sd = ts.getSchema("tap_schema"); + log.debug("testDetailMin: " + sd.getSchemaName()); + Assert.assertFalse(sd.getTableDescs().isEmpty()); + for (TableDesc td : sd.getTableDescs()) + { + Assert.assertTrue("no columns:" + td.getTableName(), td.getColumnDescs().isEmpty()); + } + } + catch(Exception unexpected) + { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } + } +} diff --git a/youcat/src/intTest/resources/AsyncErrorTest.foo.properties b/youcat/src/intTest/resources/AsyncErrorTest.foo.properties new file mode 120000 index 00000000..ca4bb89c --- /dev/null +++ b/youcat/src/intTest/resources/AsyncErrorTest.foo.properties @@ -0,0 +1 @@ +SyncErrorTest.foo.properties \ No newline at end of file diff --git a/youcat/src/intTest/resources/AsyncErrorTest.pi.properties b/youcat/src/intTest/resources/AsyncErrorTest.pi.properties new file mode 120000 index 00000000..f19d3373 --- /dev/null +++ b/youcat/src/intTest/resources/AsyncErrorTest.pi.properties @@ -0,0 +1 @@ +SyncErrorTest.pi.properties \ No newline at end of file diff --git a/youcat/src/intTest/resources/AsyncResultTest-ts-columns.properties b/youcat/src/intTest/resources/AsyncResultTest-ts-columns.properties new file mode 120000 index 00000000..2456f42f --- /dev/null +++ b/youcat/src/intTest/resources/AsyncResultTest-ts-columns.properties @@ -0,0 +1 @@ +SyncResultTest-ts-columns.properties \ No newline at end of file diff --git a/youcat/src/intTest/resources/SyncErrorTest-ATTEMPT-DELETE.properties b/youcat/src/intTest/resources/SyncErrorTest-ATTEMPT-DELETE.properties new file mode 100644 index 00000000..b9b7d758 --- /dev/null +++ b/youcat/src/intTest/resources/SyncErrorTest-ATTEMPT-DELETE.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=delete from TAP_SCHEMA.schemas where name='BAR' diff --git a/youcat/src/intTest/resources/SyncErrorTest-ATTEMPT-INSERT.properties b/youcat/src/intTest/resources/SyncErrorTest-ATTEMPT-INSERT.properties new file mode 100644 index 00000000..53370ded --- /dev/null +++ b/youcat/src/intTest/resources/SyncErrorTest-ATTEMPT-INSERT.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=insert into TAP_SCHEMA.schemas (name,description,utype) values ('FOO','foo','oops') diff --git a/youcat/src/intTest/resources/SyncErrorTest-ATTEMPT-UPDATE.properties b/youcat/src/intTest/resources/SyncErrorTest-ATTEMPT-UPDATE.properties new file mode 100644 index 00000000..75101610 --- /dev/null +++ b/youcat/src/intTest/resources/SyncErrorTest-ATTEMPT-UPDATE.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=update TAP_SCHEMA.schemas set name='FOO' where name='BAR' diff --git a/youcat/src/intTest/resources/SyncErrorTest-BAD-LANG.properties b/youcat/src/intTest/resources/SyncErrorTest-BAD-LANG.properties new file mode 100644 index 00000000..a8bb2092 --- /dev/null +++ b/youcat/src/intTest/resources/SyncErrorTest-BAD-LANG.properties @@ -0,0 +1,2 @@ +LANG=UNKNOWN LANG +QUERy=select top 1 obsID from unknownschema.unknowntable diff --git a/youcat/src/intTest/resources/SyncErrorTest-BAD-QUERY.properties b/youcat/src/intTest/resources/SyncErrorTest-BAD-QUERY.properties new file mode 100644 index 00000000..4e659259 --- /dev/null +++ b/youcat/src/intTest/resources/SyncErrorTest-BAD-QUERY.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=select from where diff --git a/youcat/src/intTest/resources/SyncErrorTest-NO-LANG.properties b/youcat/src/intTest/resources/SyncErrorTest-NO-LANG.properties new file mode 100644 index 00000000..64ef5c7c --- /dev/null +++ b/youcat/src/intTest/resources/SyncErrorTest-NO-LANG.properties @@ -0,0 +1 @@ +QUERy=select top 1 obsID from tap_schema.schemas diff --git a/youcat/src/intTest/resources/SyncErrorTest-NO-QUERY.properties b/youcat/src/intTest/resources/SyncErrorTest-NO-QUERY.properties new file mode 100644 index 00000000..06640043 --- /dev/null +++ b/youcat/src/intTest/resources/SyncErrorTest-NO-QUERY.properties @@ -0,0 +1 @@ +LANG=ADQL diff --git a/youcat/src/intTest/resources/SyncErrorTest-NO-SUCH-TABLE.properties b/youcat/src/intTest/resources/SyncErrorTest-NO-SUCH-TABLE.properties new file mode 100644 index 00000000..4ddd14f9 --- /dev/null +++ b/youcat/src/intTest/resources/SyncErrorTest-NO-SUCH-TABLE.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=select * from no_such_table diff --git a/youcat/src/intTest/resources/SyncErrorTest.foo.properties b/youcat/src/intTest/resources/SyncErrorTest.foo.properties new file mode 100644 index 00000000..64dba096 --- /dev/null +++ b/youcat/src/intTest/resources/SyncErrorTest.foo.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=SELECT foo(arraysize) from tap_schema.columns where arraysize is not null diff --git a/youcat/src/intTest/resources/SyncErrorTest.pi.properties b/youcat/src/intTest/resources/SyncErrorTest.pi.properties new file mode 100644 index 00000000..7d08111b --- /dev/null +++ b/youcat/src/intTest/resources/SyncErrorTest.pi.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=select top 1 pi() from tap_schema.schemas diff --git a/youcat/src/intTest/resources/SyncErrorTest.rand.properties b/youcat/src/intTest/resources/SyncErrorTest.rand.properties new file mode 100644 index 00000000..6dbe7837 --- /dev/null +++ b/youcat/src/intTest/resources/SyncErrorTest.rand.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=select top 1 rand() from tap_schema.schemas diff --git a/youcat/src/intTest/resources/SyncErrorTest.range.properties b/youcat/src/intTest/resources/SyncErrorTest.range.properties new file mode 100644 index 00000000..d8374bca --- /dev/null +++ b/youcat/src/intTest/resources/SyncErrorTest.range.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=select top 1 * from cfht.wideU where CONTAINS(pos, RANGE_S2D(10,11)) = 1 diff --git a/youcat/src/intTest/resources/SyncErrorTest.truncate.properties b/youcat/src/intTest/resources/SyncErrorTest.truncate.properties new file mode 100644 index 00000000..8bab3e9d --- /dev/null +++ b/youcat/src/intTest/resources/SyncErrorTest.truncate.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=select top 1 truncate(10.25), truncate(10.25, 2) from tap_schema.schemas diff --git a/youcat/src/intTest/resources/SyncResultTest-ts-columns.properties b/youcat/src/intTest/resources/SyncResultTest-ts-columns.properties new file mode 100644 index 00000000..0f5e1c44 --- /dev/null +++ b/youcat/src/intTest/resources/SyncResultTest-ts-columns.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=select count(*) from TAP_SCHEMA.columns diff --git a/youcat/src/intTest/resources/SyncResultTest-ts-keycolumns.properties b/youcat/src/intTest/resources/SyncResultTest-ts-keycolumns.properties new file mode 100644 index 00000000..6577b140 --- /dev/null +++ b/youcat/src/intTest/resources/SyncResultTest-ts-keycolumns.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=select count(*) from TAP_SCHEMA.key_columns diff --git a/youcat/src/intTest/resources/SyncResultTest-ts-keys.properties b/youcat/src/intTest/resources/SyncResultTest-ts-keys.properties new file mode 100644 index 00000000..1493b75e --- /dev/null +++ b/youcat/src/intTest/resources/SyncResultTest-ts-keys.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=select count(*) from TAP_SCHEMA.keys diff --git a/youcat/src/intTest/resources/SyncResultTest-ts-schemas.properties b/youcat/src/intTest/resources/SyncResultTest-ts-schemas.properties new file mode 100644 index 00000000..4b3b8830 --- /dev/null +++ b/youcat/src/intTest/resources/SyncResultTest-ts-schemas.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=select count(*) from TAP_SCHEMA.schemas diff --git a/youcat/src/intTest/resources/SyncResultTest-ts-tables-fqcn.properties b/youcat/src/intTest/resources/SyncResultTest-ts-tables-fqcn.properties new file mode 100644 index 00000000..99de64b2 --- /dev/null +++ b/youcat/src/intTest/resources/SyncResultTest-ts-tables-fqcn.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=select TAP_SCHEMA.tables.table_name from TAP_SCHEMA.tables diff --git a/youcat/src/intTest/resources/SyncResultTest-ts-tables.properties b/youcat/src/intTest/resources/SyncResultTest-ts-tables.properties new file mode 100644 index 00000000..a9f67a31 --- /dev/null +++ b/youcat/src/intTest/resources/SyncResultTest-ts-tables.properties @@ -0,0 +1,2 @@ +LANG=ADQL +QUERY=select count(*) from TAP_SCHEMA.tables diff --git a/youcat/src/intTest/resources/TAPUploadTest-1.xml b/youcat/src/intTest/resources/TAPUploadTest-1.xml new file mode 100644 index 00000000..e8cc56db --- /dev/null +++ b/youcat/src/intTest/resources/TAPUploadTest-1.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + +
BLASTBLASTabell31122006-12-21
MACHOMACHO-123
CFHT182543p
+ + diff --git a/youcat/src/intTest/resources/TAPUploadTest-2.xml b/youcat/src/intTest/resources/TAPUploadTest-2.xml new file mode 100644 index 00000000..cace668b --- /dev/null +++ b/youcat/src/intTest/resources/TAPUploadTest-2.xml @@ -0,0 +1,30 @@ + + + + + + + data collection this observation belongs to + + + collection-specific unique observation identifier + + + + + + + + + + + + + + + + + +
HSTHSTmisc-123
CGPSCGPS-012345
JCMT1234567890
+ + \ No newline at end of file diff --git a/youcat/src/main/java/org/opencadc/youcat/CatalogQueryRunner.java b/youcat/src/main/java/org/opencadc/youcat/CatalogQueryRunner.java new file mode 100644 index 00000000..868d19d5 --- /dev/null +++ b/youcat/src/main/java/org/opencadc/youcat/CatalogQueryRunner.java @@ -0,0 +1,106 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2018. (c) 2018. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.youcat; + + +import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.tap.QueryRunner; +import ca.nrc.cadc.uws.Job; +import javax.naming.NamingException; +import javax.sql.DataSource; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +public class CatalogQueryRunner extends QueryRunner +{ + private static final Logger log = Logger.getLogger(CatalogQueryRunner.class); + + public CatalogQueryRunner() { } + + @Override + protected DataSource getUploadDataSource() throws Exception { + return getQueryDataSource(); + } + + @Override + protected DataSource getTapSchemaDataSource() throws Exception { + return getQueryDataSource(); + } + + @Override + protected DataSource getQueryDataSource() throws Exception { + String dsName = DataSourceProviderImpl.getDataSourceNameForJob(job, "tapuser"); + return DBUtil.findJNDIDataSource(dsName); + } + + + +} diff --git a/youcat/src/main/java/org/opencadc/youcat/CatalogTapService.java b/youcat/src/main/java/org/opencadc/youcat/CatalogTapService.java new file mode 100644 index 00000000..ec46d035 --- /dev/null +++ b/youcat/src/main/java/org/opencadc/youcat/CatalogTapService.java @@ -0,0 +1,226 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2019. (c) 2019. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ + */ + +package org.opencadc.youcat; + +import ca.nrc.cadc.auth.AuthMethod; +import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.reg.Standards; +import ca.nrc.cadc.reg.client.LocalAuthority; +import ca.nrc.cadc.reg.client.RegistryClient; +import ca.nrc.cadc.rest.RestAction; +import ca.nrc.cadc.tap.schema.InitDatabaseTS; +import ca.nrc.cadc.uws.server.impl.InitDatabaseUWS; +import ca.nrc.cadc.vosi.Availability; +import ca.nrc.cadc.vosi.AvailabilityPlugin; +import ca.nrc.cadc.vosi.avail.CheckDataSource; +import ca.nrc.cadc.vosi.avail.CheckException; +import ca.nrc.cadc.vosi.avail.CheckResource; +import ca.nrc.cadc.vosi.avail.CheckWebService; +import java.io.File; +import java.net.URI; +import java.net.URL; +import javax.sql.DataSource; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +public class CatalogTapService implements AvailabilityPlugin { + + private static final Logger log = Logger.getLogger(CatalogTapService.class); + + private static final String TAPUSER_TEST = "select schema_name from tap_schema.schemas11 where schema_name='tap_schema'"; + private static final String UWS_TEST = "select jobID from uws.Job limit 1"; + + private static File SERVOPS_CERT = new File(System.getProperty("user.home") + "/.ssl/cadcproxy.pem"); + private static File TMPOPS_CERT = new File(System.getProperty("user.home") + "/.ssl/tmpops.pem"); + + private static final URI TMP_STORAGE_WS = URI.create("ivo://cadc.nrc.ca/cadc/minoc"); + + private String appName; + + public CatalogTapService() { + } + + @Override + public void setAppName(String appName) { + this.appName = appName; + } + + @Override + public Availability getStatus() { + boolean isGood = true; + String note = "service is accepting queries"; + try { + String state = getState(); + if (RestAction.STATE_OFFLINE.equals(state)) { + return new Availability(false, RestAction.STATE_OFFLINE_MSG); + } + if (RestAction.STATE_READ_ONLY.equals(state)) { + return new Availability(false, RestAction.STATE_READ_ONLY_MSG); + } + + // ReadWrite: proceed with live checks + CheckResource cr; + + String dsName = DataSourceProviderImpl.getDataSourceName(null, "tapadm"); + DataSource tapadm = DBUtil.findJNDIDataSource(dsName); + InitDatabaseTS tsi = new InitDatabaseTS(tapadm, null, "tap_schema"); + tsi.doInit(); + + dsName = DataSourceProviderImpl.getDataSourceName(null, "uws"); + DataSource uws = DBUtil.findJNDIDataSource(dsName); + InitDatabaseUWS uwsi = new InitDatabaseUWS(uws, null, "uws"); + uwsi.doInit(); + + cr = new CheckDataSource(uws, UWS_TEST); + cr.check(); + + dsName = DataSourceProviderImpl.getDataSourceName(null, "tapuser"); + cr = new CheckDataSource(dsName, TAPUSER_TEST); + cr.check(); + // TODO: check that DS_TAPUSER can create/drop TAP_UPLOAD tables + + // create/drop in all user schemas + //File cert = TODO; + //CheckCertificate checkCert = new CheckCertificate(cert); + //checkCert.check(); + + // check other services we depend on + RegistryClient reg = new RegistryClient(); + URL url; + CheckResource checkResource; + + LocalAuthority localAuthority = new LocalAuthority(); + + URI credURI = localAuthority.getServiceURI(Standards.CRED_PROXY_10.toString()); + url = reg.getServiceURL(credURI, Standards.VOSI_AVAILABILITY, AuthMethod.ANON); + checkResource = new CheckWebService(url); + checkResource.check(); + + URI usersURI = localAuthority.getServiceURI(Standards.UMS_USERS_01.toString()); + url = reg.getServiceURL(usersURI, Standards.VOSI_AVAILABILITY, AuthMethod.ANON); + checkResource = new CheckWebService(url); + checkResource.check(); + + URI groupsURI = localAuthority.getServiceURI(Standards.GMS_SEARCH_01.toString()); + if (!groupsURI.equals(usersURI)) { + url = reg.getServiceURL(groupsURI, Standards.VOSI_AVAILABILITY, AuthMethod.ANON); + checkResource = new CheckWebService(url); + checkResource.check(); + } + + url = reg.getServiceURL(TMP_STORAGE_WS, Standards.VOSI_AVAILABILITY, AuthMethod.ANON); + checkResource = new CheckWebService(url); + checkResource.check(); + + } catch (CheckException ce) { + // tests determined that the resource is not working + isGood = false; + note = ce.getMessage(); + } catch (Throwable t) { + // the test itself failed + log.error("test failed", t); + isGood = false; + note = "test failed, reason: " + t; + } + return new Availability(isGood, note); + } + + @Override + public void setState(String state) { + String key = appName + RestAction.STATE_MODE_KEY; + if (RestAction.STATE_OFFLINE.equalsIgnoreCase(state)) { + System.setProperty(key, RestAction.STATE_OFFLINE); + //} else if (RestAction.STATE_READ_ONLY.equalsIgnoreCase(state)) { + // System.setProperty(key, RestAction.STATE_READ_ONLY); + } else if (RestAction.STATE_READ_WRITE.equalsIgnoreCase(state)) { + System.setProperty(key, RestAction.STATE_READ_WRITE); + } else { + throw new IllegalArgumentException("invalid state: " + state + + " expected: " + RestAction.STATE_READ_WRITE + "|" + RestAction.STATE_OFFLINE); + } + log.info("WebService state changed: " + key + "=" + state); + } + + @Override + public boolean heartbeat() throws RuntimeException { + return true; + } + + private String getState() { + String key = appName + RestAction.STATE_MODE_KEY; + String ret = System.getProperty(key); + if (ret == null) { + return RestAction.STATE_READ_WRITE; + } + return ret; + } + +} diff --git a/youcat/src/main/java/org/opencadc/youcat/DataSourceProviderImpl.java b/youcat/src/main/java/org/opencadc/youcat/DataSourceProviderImpl.java new file mode 100644 index 00000000..32de42b4 --- /dev/null +++ b/youcat/src/main/java/org/opencadc/youcat/DataSourceProviderImpl.java @@ -0,0 +1,133 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2019. (c) 2019. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.youcat; + + +import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.uws.Job; +import ca.nrc.cadc.vosi.actions.DataSourceProvider; +import javax.naming.NamingException; +import javax.sql.DataSource; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +public class DataSourceProviderImpl extends DataSourceProvider { + private static final Logger log = Logger.getLogger(DataSourceProviderImpl.class); + + public DataSourceProviderImpl() { + } + + @Override + public DataSource getDataSource(String requestPath) { + String db = extractDatabaseFromPath(requestPath); + String dsname = getDataSourceName(db, "tapadm"); + try { + log.debug("JDNI lookup: " + dsname); + return DBUtil.findJNDIDataSource(dsname); + } catch (NamingException ex) { + throw new RuntimeException("CONFIG: failed to find datasource " + dsname, ex); + } + } + + static String getDataSourceNameForJob(Job job, String connectionType) { + String path = job.getRequestPath(); + String dbName = extractDatabaseFromPath(path); + String dsName = getDataSourceName(dbName, connectionType); + log.debug("request path: " + path + "database: " + dbName + " datasource: " + dsName); + return dsName; + } + + // used for availability checks + static String getDataSourceName(String db, String connectionType) { + StringBuilder sb = new StringBuilder(); + sb.append("jdbc/"); + if (db != null) { + sb.append(db).append("-"); + } + sb.append(connectionType); + return sb.toString(); + } + + static String extractDatabaseFromPath(String requestPath) { + // single monolithic database + // 0: empty str + // 1: service name + // 2: resource name (sync|async|tables) + return null; + + //String[] ss = requestPath.split("/"); + // 0: empty str + // 1: service name + // 2: database name + //return ss[2]; + // 3: resource name (sync|async|tables) + } +} diff --git a/youcat/src/main/java/org/opencadc/youcat/QueryJobManager.java b/youcat/src/main/java/org/opencadc/youcat/QueryJobManager.java new file mode 100644 index 00000000..4e2aa2d2 --- /dev/null +++ b/youcat/src/main/java/org/opencadc/youcat/QueryJobManager.java @@ -0,0 +1,108 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2019. (c) 2019. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.youcat; + +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.auth.IdentityManager; +import ca.nrc.cadc.uws.server.JobExecutor; +import ca.nrc.cadc.uws.server.JobPersistence; +import ca.nrc.cadc.uws.server.RandomStringGenerator; +import ca.nrc.cadc.uws.server.RequestPathJobManager; +import ca.nrc.cadc.uws.server.ThreadPoolExecutor; +import ca.nrc.cadc.uws.server.impl.PostgresJobPersistence; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +public class QueryJobManager extends RequestPathJobManager { + private static final Logger log = Logger.getLogger(QueryJobManager.class); + + private static final Long MAX_EXEC_DURATION = 8 * 3600L; // 8 hours to dump a catalog to vospace + private static final Long MAX_DESTRUCTION = 7 * 24 * 60 * 60L; // 1 week + private static final Long MAX_QUOTE = 24 * 3600L; // 24 hours since we have a threadpool with queued jobs + + public QueryJobManager() { + super(); + + IdentityManager im = AuthenticationUtil.getIdentityManager(); + // persist UWS jobs to PostgreSQL using default jdbc/uws connection pool + JobPersistence jobPersist = new PostgresJobPersistence(new RandomStringGenerator(16), im, true); + + // max threads: 6 == number of simultaneously running async queries (per + // web server), plus sync queries, plus VOSI-tables queries + JobExecutor jobExec = new ThreadPoolExecutor(jobPersist, CatalogQueryRunner.class, 6); + + super.setJobPersistence(jobPersist); + super.setJobExecutor(jobExec); + super.setMaxExecDuration(MAX_EXEC_DURATION); + super.setMaxDestruction(MAX_DESTRUCTION); + super.setMaxQuote(MAX_QUOTE); + } +} diff --git a/youcat/src/main/java/org/opencadc/youcat/TableUpdateJobManager.java b/youcat/src/main/java/org/opencadc/youcat/TableUpdateJobManager.java new file mode 100644 index 00000000..c7ae2553 --- /dev/null +++ b/youcat/src/main/java/org/opencadc/youcat/TableUpdateJobManager.java @@ -0,0 +1,107 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2019. (c) 2019. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.youcat; + +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.auth.IdentityManager; +import ca.nrc.cadc.uws.server.JobExecutor; +import ca.nrc.cadc.uws.server.JobPersistence; +import ca.nrc.cadc.uws.server.RequestPathJobManager; +import ca.nrc.cadc.uws.server.ThreadPoolExecutor; + +import ca.nrc.cadc.uws.server.impl.PostgresJobPersistence; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +public class TableUpdateJobManager extends RequestPathJobManager { + private static final Logger log = Logger.getLogger(TableUpdateJobManager.class); + + private static final Long MAX_EXEC_DURATION = 4 * 3600L; // 4 hours + private static final Long MAX_DESTRUCTION = 7 * 24 * 60 * 60L; // 1 week + private static final Long MAX_QUOTE = 24 * 3600L; // 24 hours: small thread pool + + public TableUpdateJobManager() { + super(); + + IdentityManager im = AuthenticationUtil.getIdentityManager(); + // persist UWS jobs to PostgreSQL using default jdbc/uws connection pool + JobPersistence jobPersist = new PostgresJobPersistence(im); + + // max threads: 3 == number of simultaneously running jobs (per web server) + JobExecutor jobExec = new ThreadPoolExecutor(jobPersist, TableUpdateRunnerImpl.class, 3); + + super.setJobPersistence(jobPersist); + super.setJobExecutor(jobExec); + super.setMaxExecDuration(MAX_EXEC_DURATION); + super.setMaxDestruction(MAX_DESTRUCTION); + super.setMaxQuote(MAX_QUOTE); + } +} diff --git a/youcat/src/main/java/org/opencadc/youcat/TableUpdateRunnerImpl.java b/youcat/src/main/java/org/opencadc/youcat/TableUpdateRunnerImpl.java new file mode 100644 index 00000000..3c049211 --- /dev/null +++ b/youcat/src/main/java/org/opencadc/youcat/TableUpdateRunnerImpl.java @@ -0,0 +1,98 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2018. (c) 2018. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.youcat; + +import ca.nrc.cadc.db.DBUtil; +import ca.nrc.cadc.vosi.actions.TableUpdateRunner; +import javax.naming.NamingException; +import javax.sql.DataSource; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +public class TableUpdateRunnerImpl extends TableUpdateRunner { + private static final Logger log = Logger.getLogger(TableUpdateRunnerImpl.class); + + public TableUpdateRunnerImpl() { + } + + @Override + protected DataSource getDataSource() { + String dsName = DataSourceProviderImpl.getDataSourceNameForJob(job, "tapadm"); + try { + log.debug("JNDI lookup: " + dsName); + return DBUtil.findJNDIDataSource(dsName); + } catch (NamingException ex) { + throw new RuntimeException("CONFIG: failed to find datasource " + dsName, ex); + } + } + + +} diff --git a/youcat/src/main/java/org/opencadc/youcat/tap/AdqlQueryImpl.java b/youcat/src/main/java/org/opencadc/youcat/tap/AdqlQueryImpl.java new file mode 100644 index 00000000..25de8d35 --- /dev/null +++ b/youcat/src/main/java/org/opencadc/youcat/tap/AdqlQueryImpl.java @@ -0,0 +1,143 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2018. (c) 2018. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.youcat.tap; + +import ca.nrc.cadc.auth.AuthenticationUtil; +import ca.nrc.cadc.auth.IdentityManager; +import ca.nrc.cadc.tap.AdqlQuery; +import ca.nrc.cadc.tap.permissions.TapSchemaReadAccessConverter; +import ca.nrc.cadc.tap.parser.BaseExpressionDeParser; +import ca.nrc.cadc.tap.parser.PgsphereDeParser; +import ca.nrc.cadc.tap.parser.converter.TableNameConverter; +import ca.nrc.cadc.tap.parser.converter.TableNameReferenceConverter; +import ca.nrc.cadc.tap.parser.converter.TopConverter; +import ca.nrc.cadc.tap.parser.converter.postgresql.PgFunctionNameConverter; +import ca.nrc.cadc.tap.parser.extractor.FunctionExpressionExtractor; +import ca.nrc.cadc.tap.parser.navigator.ExpressionNavigator; +import ca.nrc.cadc.tap.parser.navigator.FromItemNavigator; +import ca.nrc.cadc.tap.parser.navigator.ReferenceNavigator; +import ca.nrc.cadc.tap.parser.navigator.SelectNavigator; +import ca.nrc.cadc.tap.parser.region.pgsphere.PgsphereRegionConverter; +import net.sf.jsqlparser.util.deparser.SelectDeParser; +import org.apache.log4j.Logger; + +/** + * AdqlQuery implementation for PostgreSQL + pg-sphere and arbitrary catalogue tables. + * + * @author pdowler + */ +public class AdqlQueryImpl extends AdqlQuery { + + private static Logger log = Logger.getLogger(AdqlQueryImpl.class); + + public AdqlQueryImpl() { + } + + @Override + protected void init() { + super.init(); + + // convert TOP -> LIMIT + super.navigatorList.add(new TopConverter( + new ExpressionNavigator(), new ReferenceNavigator(), new FromItemNavigator())); + + // convert some ADQL function calls to alternate column refs + super.navigatorList.add(new PgsphereRegionConverter( + new ExpressionNavigator(), new ReferenceNavigator(), new FromItemNavigator())); + + super.navigatorList.add(new FunctionExpressionExtractor( + new PgFunctionNameConverter(), new ReferenceNavigator(), new FromItemNavigator())); + + // inject read access constraints in tap_schema queries + IdentityManager identityManager = AuthenticationUtil.getIdentityManager(); + super.navigatorList.add(new TapSchemaReadAccessConverter(identityManager)); + + // TAP-1.1 version of tap_schema + TableNameConverter tnc = new TableNameConverter(true); + tnc.put("tap_schema.schemas", "tap_schema.schemas11"); + tnc.put("tap_schema.tables", "tap_schema.tables11"); + tnc.put("tap_schema.columns", "tap_schema.columns11"); + tnc.put("tap_schema.keys", "tap_schema.keys11"); + tnc.put("tap_schema.key_columns", "tap_schema.key_columns11"); + TableNameReferenceConverter tnrc = new TableNameReferenceConverter(tnc.map); + super.navigatorList.add(new SelectNavigator(new ExpressionNavigator(), tnrc, tnc)); + + } + + @Override + protected BaseExpressionDeParser getExpressionDeparser(SelectDeParser dep, StringBuffer sb) { + return new PgsphereDeParser(dep, sb); + } + + @Override + public String getSQL() { + String sql = super.getSQL(); + log.debug("SQL:\n" + sql); + return sql; + } +} diff --git a/youcat/src/main/java/org/opencadc/youcat/tap/FormatFactoryImpl.java b/youcat/src/main/java/org/opencadc/youcat/tap/FormatFactoryImpl.java new file mode 100644 index 00000000..cbfd0162 --- /dev/null +++ b/youcat/src/main/java/org/opencadc/youcat/tap/FormatFactoryImpl.java @@ -0,0 +1,137 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2018. (c) 2018. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.youcat.tap; + +import ca.nrc.cadc.dali.util.Format; +import ca.nrc.cadc.tap.TapSelectItem; +import ca.nrc.cadc.tap.pg.IntervalFormat; +import ca.nrc.cadc.tap.writer.format.DefaultFormatFactory; +import ca.nrc.cadc.tap.writer.format.SCircleFormat; +import ca.nrc.cadc.tap.writer.format.SPointFormat; +import ca.nrc.cadc.tap.writer.format.SPointFormat10; +import ca.nrc.cadc.tap.writer.format.SPolyFormat; +import ca.nrc.cadc.tap.writer.format.SPolyFormat10; +import org.apache.log4j.Logger; + +/** + * + * + */ +public class FormatFactoryImpl extends DefaultFormatFactory { + + private static Logger log = Logger.getLogger(FormatFactoryImpl.class); + + public FormatFactoryImpl() { + super(); + } + + @Override + public Format getFormat(TapSelectItem d) { + Format ret = super.getFormat(d); + log.debug("fomatter: " + d + " " + ret.getClass().getName()); + return ret; + } + + @Override + public Format getIntervalFormat(TapSelectItem columnDesc) { + return new IntervalFormat(columnDesc.getDatatype().isVarSize()); + } + + @Override + public Format getPointFormat(TapSelectItem columnDesc) { + log.debug("getPointFormat: " + columnDesc); + return new SPointFormat(); + } + + @Override + public Format getCircleFormat(TapSelectItem columnDesc) { + log.debug("getCircleFormat: " + columnDesc); + return new SCircleFormat(); + } + + @Override + protected Format getPolygonFormat(TapSelectItem columnDesc) { + log.debug("getPolygonFormat: " + columnDesc); + //if (columnDesc.utype != null && columnDesc.utype.equals("caom2:Plane.position.bounds")) + // return new DoubleArrayFormat(); // see CaomSelectListConverter + return new SPolyFormat(); + } + + @Override + protected Format getPositionFormat(TapSelectItem columnDesc) { + log.debug("getPositionFormat: " + columnDesc); + return new SPointFormat10(); + } + + @Override + public Format getRegionFormat(TapSelectItem columnDesc) { + log.debug("getRegionFormat: " + columnDesc); + return new SPolyFormat10(); + } + +} diff --git a/youcat/src/main/java/org/opencadc/youcat/tap/MaxRecValidatorImpl.java b/youcat/src/main/java/org/opencadc/youcat/tap/MaxRecValidatorImpl.java new file mode 100644 index 00000000..f38ad17a --- /dev/null +++ b/youcat/src/main/java/org/opencadc/youcat/tap/MaxRecValidatorImpl.java @@ -0,0 +1,169 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2018. (c) 2018. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.youcat.tap; + +import ca.nrc.cadc.tap.AdqlQuery; +import ca.nrc.cadc.tap.DefaultTableWriter; +import ca.nrc.cadc.tap.MaxRecValidator; +import ca.nrc.cadc.tap.parser.extractor.SelectListExtractor; +import ca.nrc.cadc.tap.parser.navigator.SelectNavigator; +import ca.nrc.cadc.util.StringUtil; +import ca.nrc.cadc.uws.ParameterUtil; +import java.util.List; +import org.apache.log4j.Logger; + +/** + * + * @author pdowler + */ +public class MaxRecValidatorImpl extends MaxRecValidator { + + private static Logger log = Logger.getLogger(MaxRecValidatorImpl.class); + + // HACK: assume UTF-8 and mainly short strings + // or numbers with ~8 decimal places + private static final double BYTES_PER_COLUMN = 26.0; + + private static final double XML_BLOAT = 2.4; + + private static final long MAX_FILE_SIZE = 128 * 1024 * 1024; // 128 MB + + public MaxRecValidatorImpl() { + super(); + } + + @Override + public Integer validate() { + if (sync) { + // user specified limit only + Integer maxrec = super.validate(); + log.debug("sync: maxrec=" + maxrec); + return maxrec; + } + + // async -> vospace: no enforced limit + try { + String destinationValue = ParameterUtil.findParameterValue("DEST", job.getParameterList()); + if (StringUtil.hasText(destinationValue) + && destinationValue.startsWith("vos://")) { + Integer maxrec = super.validate(); + log.debug("async-vospace: maxrec=" + maxrec); + return maxrec; + } + + AdqlQueryHack adql = new AdqlQueryHack(); + adql.setJob(job); + adql.setTapSchema(tapSchema); + int numCols = adql.getSelectListSize(); + double rowSize = numCols * BYTES_PER_COLUMN; + + String format = ParameterUtil.findParameterValue("FORMAT", job.getParameterList()); + if (DefaultTableWriter.VOTABLE.equals(format)) { + rowSize *= XML_BLOAT; + } + int numRows = (int) (MAX_FILE_SIZE / rowSize); + setDefaultValue(numRows); + setMaxValue(numRows); + Integer maxrec = super.validate(); + log.debug("numCols=" + numCols + " rowSize=" + rowSize + " numRows=" + numRows + " dynamic async maxrec=" + maxrec); + return maxrec; + } finally { + setDefaultValue(null); + setMaxValue(null); + } + } + + private class AdqlQueryHack extends AdqlQuery { + + AdqlQueryHack() { + super(); + } + + @Override + protected void init() { + super.init(); + if (tapSchema == null) // unit test mode + { + SelectNavigator keep = null; + for (SelectNavigator sn : super.navigatorList) { + if (sn instanceof SelectListExtractor) { + keep = sn; + } + } + super.navigatorList.clear(); + super.navigatorList.add(keep); + } + } + + int getSelectListSize() { + List items = super.getSelectList(); + return items.size(); + } + } + +} diff --git a/youcat/src/main/java/org/opencadc/youcat/tap/TapSchemaDAOImpl.java b/youcat/src/main/java/org/opencadc/youcat/tap/TapSchemaDAOImpl.java new file mode 100644 index 00000000..138c0d62 --- /dev/null +++ b/youcat/src/main/java/org/opencadc/youcat/tap/TapSchemaDAOImpl.java @@ -0,0 +1,138 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2011. (c) 2011. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +* $Revision: 5 $ +* +************************************************************************ + */ + +package org.opencadc.youcat.tap; + +import ca.nrc.cadc.cred.client.CredUtil; +import ca.nrc.cadc.net.ResourceNotFoundException; +import ca.nrc.cadc.reg.Standards; +import ca.nrc.cadc.reg.client.LocalAuthority; +import ca.nrc.cadc.tap.schema.FunctionDesc; +import ca.nrc.cadc.tap.schema.TapDataType; +import ca.nrc.cadc.tap.schema.TapSchemaDAO; +import java.io.IOException; +import java.net.URI; +import java.security.AccessControlException; +import java.security.cert.CertificateException; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import org.apache.log4j.Logger; +import org.opencadc.gms.GroupURI; +import org.opencadc.gms.IvoaGroupClient; + +/** + * + * @author pdowler + */ +public class TapSchemaDAOImpl extends TapSchemaDAO { + + private static final Logger log = Logger.getLogger(TapSchemaDAOImpl.class); + + public TapSchemaDAOImpl() { + super(); + } + + private IvoaGroupClient gmsClient; + + // testing support + void setGMSClient(IvoaGroupClient gmsClient) { + this.gmsClient = gmsClient; + } + + @Override + protected List getFunctionDescs() { + List ret = super.getFunctionDescs(); + ret.add(new FunctionDesc("RANGE_S2D", new TapDataType("double", "4", "range"))); + ret.add(new FunctionDesc("NOW", TapDataType.TIMESTAMP)); + return ret; + } + + public static Set getGroupIDs(IvoaGroupClient gmsClient) + throws AccessControlException, InterruptedException, ResourceNotFoundException { + try { + if (CredUtil.checkCredentials()) { + IvoaGroupClient gms = gmsClient; + if (gms == null) { + gms = new IvoaGroupClient(); + } + LocalAuthority loc = new LocalAuthority(); + URI gmsURI = loc.getServiceURI(Standards.GMS_SEARCH_10.toString()); + Set groups = gms.getMemberships(gmsURI); + log.debug("search " + gmsURI + "found: " + groups.size()); + return groups; + } + throw new AccessControlException("cannot check group memberships - no credentials available"); + } catch (CertificateException ex) { + throw new RuntimeException("failed to find group memberships (invalid proxy certficate)", ex); + } catch (IOException ex) { + throw new RuntimeException("failed to find group memberships", ex); + } + } +} diff --git a/youcat/src/main/java/org/opencadc/youcat/tap/UploadManagerImpl.java b/youcat/src/main/java/org/opencadc/youcat/tap/UploadManagerImpl.java new file mode 100644 index 00000000..2813cc2a --- /dev/null +++ b/youcat/src/main/java/org/opencadc/youcat/tap/UploadManagerImpl.java @@ -0,0 +1,109 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2018. (c) 2018. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +************************************************************************ +*/ + +package org.opencadc.youcat.tap; + +import ca.nrc.cadc.stc.Polygon; +import ca.nrc.cadc.stc.Position; +import ca.nrc.cadc.stc.Region; +import ca.nrc.cadc.tap.BasicUploadManager; +import ca.nrc.cadc.tap.parser.region.pgsphere.function.Spoint; +import ca.nrc.cadc.tap.parser.region.pgsphere.function.Spoly; +import java.sql.SQLException; +import org.postgresql.util.PGobject; + +/** + * + * @author pdowler + */ +public class UploadManagerImpl extends BasicUploadManager { + + /** + * Default maximum number of rows allowed in the UPLOAD VOTable. + */ + public static final int MAX_UPLOAD_ROWS = 10000; + + public UploadManagerImpl() { + super(MAX_UPLOAD_ROWS); + } + + /** + * Create the SQL to grant select privileges to the cvopub group for the + * specified table. + * + * @param databaseTableName fully qualified table name. + * @return SQL to grant select privileges to the cvopub group. + */ + @Override + protected String getGrantSelectTableSQL(String databaseTableName) { + StringBuilder sb = new StringBuilder(); + sb.append("grant select on table "); + sb.append(databaseTableName); + sb.append(" to group cvopub"); + return sb.toString(); + } +} diff --git a/youcat/src/main/resources/PluginFactory.properties b/youcat/src/main/resources/PluginFactory.properties new file mode 100644 index 00000000..07d5f6fd --- /dev/null +++ b/youcat/src/main/resources/PluginFactory.properties @@ -0,0 +1,25 @@ + +## commented out values are the defaults, shown as examples +## to customise behaviour, subclass the specified class and +## change the configuration here + +# configure supported values of LANG parameter and the TapQuery implementation +ca.nrc.cadc.tap.TapQuery.langValues = ADQL ADQL-2.0 +ADQL = org.opencadc.youcat.tap.AdqlQueryImpl +ADQL-2.0 = org.opencadc.youcat.tap.AdqlQueryImpl + +ca.nrc.cadc.tap.MaxRecValidator = org.opencadc.youcat.tap.MaxRecValidatorImpl + +ca.nrc.cadc.tap.db.DatabaseDataType=ca.nrc.cadc.tap.pg.PostgresDataTypeMapper + +ca.nrc.cadc.tap.UploadManager = org.opencadc.youcat.tap.UploadManagerImpl + +#ca.nrc.cadc.tap.TableWriter = ca.nrc.cadc.tap.DefaultTableWriter + +ca.nrc.cadc.tap.writer.format.FormatFactory = org.opencadc.youcat.tap.FormatFactoryImpl + +ca.nrc.cadc.tap.ResultStore = org.opencadc.tap.tmp.HttpStorageManager + +ca.nrc.cadc.tap.schema.TapSchemaDAO = org.opencadc.youcat.tap.TapSchemaDAOImpl + +ca.nrc.cadc.vosi.actions.DataSourceProvider = org.opencadc.youcat.DataSourceProviderImpl \ No newline at end of file diff --git a/youcat/src/main/webapp/META-INF/context.xml b/youcat/src/main/webapp/META-INF/context.xml new file mode 100644 index 00000000..2c1bb428 --- /dev/null +++ b/youcat/src/main/webapp/META-INF/context.xml @@ -0,0 +1,41 @@ + + + + WEB-INF/web.xml + + + + + + + diff --git a/youcat/src/main/webapp/WEB-INF/web.xml b/youcat/src/main/webapp/WEB-INF/web.xml new file mode 100644 index 00000000..2a5bd68c --- /dev/null +++ b/youcat/src/main/webapp/WEB-INF/web.xml @@ -0,0 +1,298 @@ + + + + + youcat + + index.html + + + + logControl + ca.nrc.cadc.log.LogControlServlet + + logLevel + info + + + logLevelPackages + + org.opencadc.tap + ca.nrc.cadc.rest + ca.nrc.cadc.uws + ca.nrc.cadc.dali + ca.nrc.cadc.tap + ca.nrc.cadc.db + ca.nrc.cadc.cat + ca.nrc.cadc.net + ca.nrc.cadc.auth + ca.nrc.cadc.vosi + + + + logAccessGroup + ivo://cadc.nrc.ca/gms?CADC + + + groupAuthorizer + ca.nrc.cadc.ac.client.GroupAuthorizer + + + logControlProperties + youcat-logControl.properties + + 1 + + + + AsyncQueryServlet + ca.nrc.cadc.uws.server.JobServlet + + ca.nrc.cadc.uws.server.JobManager + ca.nrc.cadc.cat.ws.QueryJobManager + + + ca.nrc.cadc.rest.InlineContentHandler + org.opencadc.tap.tmp.HttpStorageManager + + + get + ca.nrc.cadc.uws.web.GetAction + + + post + ca.nrc.cadc.uws.web.PostAction + + + delete + ca.nrc.cadc.uws.web.DeleteAction + + 2 + + + + SyncQueryServlet + ca.nrc.cadc.uws.server.JobServlet + + ca.nrc.cadc.uws.server.JobManager + ca.nrc.cadc.cat.ws.QueryJobManager + + + ca.nrc.cadc.rest.InlineContentHandler + org.opencadc.tap.tmp.HttpStorageManager + + + get + ca.nrc.cadc.uws.web.SyncGetAction + + + post + ca.nrc.cadc.uws.web.SyncPostAction + + 2 + + + + CapabilitiesServlet + ca.nrc.cadc.rest.RestServlet + + init + ca.nrc.cadc.vosi.CapInitAction + + + get + ca.nrc.cadc.vosi.CapGetAction + + + head + ca.nrc.cadc.vosi.CapHeadAction + + + input + /capabilities.xml + + 3 + + + + AvailabilityServlet + ca.nrc.cadc.vosi.AvailabilityServlet + + + + ca.nrc.cadc.vosi.AvailabilityPlugin + ca.nrc.cadc.cat.ws.CatalogTapService + + + availabilityProperties + youcat-availability.properties + + 3 + + + + TableServlet + ca.nrc.cadc.rest.RestServlet + + get + ca.nrc.cadc.vosi.actions.GetAction + + + put + ca.nrc.cadc.vosi.actions.PutAction + + + delete + ca.nrc.cadc.vosi.actions.DeleteAction + + + 3 + + + + AsyncTableUpdateServlet + ca.nrc.cadc.uws.server.JobServlet + + ca.nrc.cadc.uws.server.JobManager + ca.nrc.cadc.cat.ws.TableUpdateJobManager + + + get + ca.nrc.cadc.uws.web.GetAction + + + post + ca.nrc.cadc.uws.web.PostAction + + + delete + ca.nrc.cadc.uws.web.DeleteAction + + 3 + + + + PermissionsServlet + ca.nrc.cadc.rest.RestServlet + + get + ca.nrc.cadc.vosi.actions.GetPermissionsAction + + + post + ca.nrc.cadc.vosi.actions.PostPermissionsAction + + 3 + + + + SyncLoadServlet + ca.nrc.cadc.rest.RestServlet + + post + ca.nrc.cadc.vosi.actions.SyncLoadAction + + 3 + + + + AsyncQueryServlet + /async/* + + + AsyncQueryServlet + /auth-async/* + + + SyncQueryServlet + /sync/* + + + SyncQueryServlet + /auth-sync/* + + + TableServlet + /tables/* + + + TableServlet + /auth-tables/* + + + AsyncTableUpdateServlet + /table-update/* + + + AsyncTableUpdateServlet + /auth-table-update/* + + + PermissionsServlet + /permissions/* + + + PermissionsServlet + /auth-permissions/* + + + SyncLoadServlet + /load/* + + + SyncLoadServlet + /auth-load/* + + + + + CapabilitiesServlet + /capabilities + + + logControl + /logControl/* + + + + AvailabilityServlet + /availability + + + + + + auth + + /auth-async/* + /auth-sync/* + /auth-tables/* + /auth-table-update/* + /auth-load/* + GET + POST + HEAD + PUT + OPTIONS + TRACE + DELETE + + + force authentication for all requests + public + + + + + BASIC + Canadian Astronomy Data Centre + + + diff --git a/youcat/src/main/webapp/capabilities.xml b/youcat/src/main/webapp/capabilities.xml new file mode 100644 index 00000000..c472d5c2 --- /dev/null +++ b/youcat/src/main/webapp/capabilities.xml @@ -0,0 +1,173 @@ + + + + + + https://example.net/youcat/capabilities + + + + + + https://example.net/youcat/availability + + + + + + https://example.net/youcat/logControl + + + + + + + + https://example.net/youcat/tables + + + + + + + https://example.net/youcat/auth-tables + + + + + + + + https://example.net/youcat/table-update + + + + + + https://example.net/youcat/auth-table-update + + + + + + + + https://example.net/youcat/load + + + + + + https://example.net/youcat/auth-load + + + + + + + + https://example.net/youcat/permissions + + + + + + https://example.net/youcat/auth-permissions + + + + + + + + https://example.net/youcat + + + + + + + + ADQL + 2.0 + ADQL-2.0 + + +
POINT
+ + +
CIRCLE
+ + +
POLYGON
+ + +
CONTAINS
+ + +
INTERSECTS
+ + +
COORD1
+ + +
COORD2
+ + + + + + + application/x-votable+xml + votable + + + + text/xml + + + + text/csv + csv + + + + text/tab-separated-values + tsv + + + + + + + + 604800 + 604800 + + + + 600 + 600 + + + + + 134217728 + 134217728 + + + + + 10000 + 10000 + + + + + + diff --git a/youcat/src/main/webapp/index.html b/youcat/src/main/webapp/index.html new file mode 100644 index 00000000..82bfb75b --- /dev/null +++ b/youcat/src/main/webapp/index.html @@ -0,0 +1,165 @@ + + + + + Swagger UI + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
 
+
+ + diff --git a/youcat/src/main/webapp/service.json b/youcat/src/main/webapp/service.json new file mode 100644 index 00000000..8470d3d9 --- /dev/null +++ b/youcat/src/main/webapp/service.json @@ -0,0 +1,772 @@ +{ + "swagger": "2.0", + "info": { + "version": "1.0", + "title": "YouCat (User Catalogue) web service", + "description": "This page contains technical API information.\n\nThis service implements the IVOA TAP-1.1 (Proposed) Recommendation plus extensions to support table creation and bulk loading of content.\n\nThe endpoints listed below support both anonymous and authenticated access (using client certificate or cookie). To use HTTP authentication (username and password) directly with the service, you must prepend \"auth-\" to the name of the resource. For example, /youcat/auth-sync for synchronous query execution, /youcat/auth-tables for table operations, and /youcat/auth-table-update for updates like creating an index.\n\n YouCat supports user management of permissions at both the schema and table level. A schema owner is set when a YouCat allocation is created by the service operator. Table owners are set to be the user who created the table. Owners can set a flag for anonymous reading and can set groups for read-only and read-write permissons.\n The following describes how the permissions are enforced on schemas and tables:\n\n \t{schema|table}.public = false|true\n \t\tallows querying via TAP API\n \t{schema|table|.read-only = NULL | {group URI}\n \t\tread-only permission on a schema: allows TAP API to expose table\n \t\tread-only permission on table: allows TAP API to expose content of a table\n \t{schema|table}.read-write = NULL | {group URI}\n \t\tread-write permissions on a schema: read-only + create table\n \t\tread-write permission on a table: read-only + create index, append rows, update (tap_schema) metadata\n \t{schema|table}.owner\n \t\towner of a schema has: read-write + drop (any) table\n \t\towner of a table has: read-write + drop table\n \t\towner of a schema: change permissions on schema and all tables\n \t\towner of a table: change permissions on table\n\nGroup URIs are in the format:\n\n \tivo://cadc.nrc.ca/gms?groupName" + }, + "schemes": [ + "https" + ], + "basePath": "/youcat", + "paths": { + "/async": { + "post": { + "summary": "IVOA TAP v1.1", + "tags": [ + "TAP" + ], + "description": "TAP asynchronous query endpoint (create UWS Job)\n", + "parameters": [ + { + "name": "LANG", + "in": "query", + "description": "specify the query language used in the QUERY parameter", + "required": true, + "type": "string" + }, + { + "name": "QUERY", + "in": "query", + "description": "specify the query", + "required": true, + "type": "string" + }, + { + "name": "FORMAT", + "in": "query", + "description": "supported for backwards compatibility to 1.0 (see: RESPONSEFORMAT)", + "required": false, + "type": "string" + }, + { + "name": "RESPONSEFORMAT", + "in": "query", + "description": "select output table format", + "required": false, + "type": "string" + }, + { + "name": "MAXREC", + "in": "query", + "description": "request a specific limit on number of rows to return", + "required": false, + "type": "integer", + "format": "int64" + }, + { + "name": "DEST", + "in": "query", + "description": "specify destination where output should be written (VOSpace URI)", + "required": false, + "type": "string", + "format": "uri" + }, + { + "name": "UPLOAD", + "in": "query", + "collectionFormat": "multi", + "description": "specify name,location pair for a table to be uploaded and used in the query", + "required": false, + "type": "string" + } + ], + "produces": [ + "text/xml" + ], + "responses": { + "200": { + "description": "Successful response" + }, + "303": { + "description": "standard UWS redirect to the created job" + }, + "401": { + "description": "Unauthorized - User not authenticated" + }, + "403": { + "description": "Permission Denied - Table(s) not readable" + }, + "404": { + "description": "Not Found - Table(s) not found" + }, + "500": { + "description": "Internal error" + }, + "503": { + "description": "Service busy" + }, + "default": { + "description": "Unexpeced error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + } + }, + "/sync": { + "get": { + "summary": "IVOA TAP v1.1", + "tags": [ + "TAP" + ], + "description": "TAP synchronous query endpoint\n", + "parameters": [ + { + "name": "LANG", + "in": "query", + "description": "specify the query language used in the QUERY parameter", + "required": true, + "type": "string" + }, + { + "name": "QUERY", + "in": "query", + "description": "specify the query", + "required": true, + "type": "string" + }, + { + "name": "FORMAT", + "in": "query", + "description": "supported for backwards compatibility to 1.0 (see: RESPONSEFORMAT)", + "required": false, + "type": "string" + }, + { + "name": "RESPONSEFORMAT", + "in": "query", + "description": "select output table format", + "required": false, + "type": "string" + }, + { + "name": "MAXREC", + "in": "query", + "description": "request a specific limit on number of rows to return", + "required": false, + "type": "integer", + "format": "int64" + }, + { + "name": "UPLOAD", + "in": "query", + "collectionFormat": "multi", + "description": "specify name,location pair for a table to be uploaded and used in the query", + "required": false, + "type": "string" + } + ], + "produces": [ + "application/x-votable+/xml" + ], + "responses": { + "200": { + "description": "Successful response" + }, + "401": { + "description": "Unauthorized - User not authenticated" + }, + "403": { + "description": "Permission Denied - Table(s) not readable" + }, + "404": { + "description": "Not Found - Table(s) not found" + }, + "500": { + "description": "Internal error" + }, + "503": { + "description": "Service busy" + }, + "default": { + "description": "Unexpeced error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + }, + "post": { + "summary": "IVOA TAP v1.1", + "tags": [ + "TAP" + ], + "description": "TAP synchronous query endpoint\n", + "parameters": [ + { + "name": "LANG", + "in": "query", + "description": "specify the query language used in the QUERY parameter", + "required": true, + "type": "string" + }, + { + "name": "QUERY", + "in": "query", + "description": "specify the query", + "required": true, + "type": "string" + }, + { + "name": "FORMAT", + "in": "query", + "description": "supported for backwards compatibility to 1.0 (see: RESPONSEFORMAT)", + "required": false, + "type": "string" + }, + { + "name": "RESPONSEFORMAT", + "in": "query", + "description": "select output table format", + "required": false, + "type": "string" + }, + { + "name": "MAXREC", + "in": "query", + "description": "request a specific limit on number of rows to return", + "required": false, + "type": "integer", + "format": "int64" + }, + { + "name": "UPLOAD", + "in": "query", + "collectionFormat": "multi", + "description": "specify name,location pair for a table to be uploaded and used in the query", + "required": false, + "type": "string" + } + ], + "produces": [ + "application/x-votable+/xml" + ], + "responses": { + "200": { + "description": "Successful response" + }, + "401": { + "description": "Unauthorized - User not authenticated" + }, + "403": { + "description": "Permission denied - Table(s) not readable" + }, + "404": { + "description": "Not Found - Table(s) not found" + }, + "500": { + "description": "Internal error" + }, + "503": { + "description": "Service busy" + }, + "default": { + "description": "Unexpeced error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + } + }, + "/availability": { + "get": { + "tags": [ + "Support Interfaces" + ], + "summary": "VOSI Availability", + "description": "Indicates whether the service is operable and shows the reliability of the service for extended and scheduled requests. If the query parameter 'detail=min' is used, a light weight heart beat test will be performed. The heart beat test returns status 200 if the service is available.", + "parameters": [ + { + "name": "detail", + "in": "query", + "description": "specifies heart beat to be used to check for availability of this service, the value 'min' must be used, otherwise the full availability test will be performed", + "required": false, + "type": "string" + } + ], + "produces": [ + "text/xml" + ], + "responses": { + "200": { + "description": "A VOSI availability document in XML.", + "schema": { + "$ref": "#/definitions/availability" + } + }, + "default": { + "description": "Unexpected error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + } + }, + "/capabilities": { + "get": { + "summary": "VOSI Capabilities", + "tags": [ + "Support Interfaces" + ], + "description": "Provides the service metadata in the form of a list of Capability descriptions. Each of these descriptions is an \nXML element that:\n
    \n
  • states that the service provides a particular, IVOA-standard function;
    • \n
  • lists the interfaces for invoking that function;
    • \n
  • records any details of the implementation of the function that are not defined as default or constant in the standard for that function.
    • \n
\n", + "produces": [ + "text/xml" + ], + "responses": { + "200": { + "description": "A VOSI Capabilities document in XML.", + "schema": { + "$ref": "#/definitions/capabilities" + } + }, + "500": { + "description": "Internal server error" + }, + "503": { + "description": "Service too busy" + }, + "default": { + "description": "Unexpected error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + } + }, + "/tables": { + "get": { + "summary": "VOSI Tables", + "tags": [ + "Support Interfaces" + ], + "description": "Provides the table metadata in the form of a TableSet descriptions.\n", + "produces": [ + "text/xml" + ], + "responses": { + "200": { + "description": "A VOSI document in XML.", + "schema": { + "$ref": "#/definitions/tables" + } + }, + "500": { + "description": "Internal server error" + }, + "503": { + "description": "Service too busy" + }, + "default": { + "description": "Unexpected error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + } + }, + "/tables/{name}": { + "get": { + "summary": "get table metadata (VOSI)", + "tags": [ + "Support Interfaces" + ], + "description": "Provides the table metadata for a single table.\n", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "a single fully qualified table name (e.g. tap_schema.tables)", + "required": true, + "type": "string" + } + ], + "produces": [ + "text/xml" + ], + "responses": { + "200": { + "description": "A VOSI document in XML.", + "schema": { + "$ref": "#/definitions/tables" + } + }, + "500": { + "description": "Internal server error" + }, + "503": { + "description": "Service too busy" + }, + "default": { + "description": "Unexpected error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + }, + "put": { + "summary": "create table", + "description": "table description in VOSI table or VOTable (no rows) format\n", + "tags": [ + "Custom Features" + ], + "parameters": [ + { + "name": "name", + "in": "path", + "description": "the fully qualified name of the table to create\n", + "required": true, + "type": "string" + } + ], + "consumes": [ + "text/xml", + "application/x-votable+xml" + ], + "responses": { + "201": { + "description": "table created" + }, + "400": { + "description": "create table failed due to invalid input" + }, + "500": { + "description": "Internal server error" + }, + "503": { + "description": "Service too busy" + }, + "default": { + "description": "Unexpected error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + }, + "delete": { + "summary": "drop table", + "tags": [ + "Custom Features" + ], + "parameters": [ + { + "name": "name", + "in": "path", + "description": "the table to drop", + "required": true, + "type": "string" + } + ], + "responses": { + "200": { + "description": "table dropped" + }, + "500": { + "description": "Internal server error" + }, + "503": { + "description": "Service too busy" + }, + "default": { + "description": "Unexpected error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + } + }, + "/permissions/{name}": { + "get": { + "summary": "view schema or table permissions", + "tags": [ + "Custom Features" + ], + "description": "View the owner, public flag, read group, and write group of a schema or table.\n", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "name of the schema or table to view.", + "required": true, + "type": "string" + } + ], + "provides": [ + "text/plain" + ], + "responses": { + "200": { + "description": "successful request" + }, + "400": { + "description": "invalid syntax" + }, + "500": { + "description": "Internal server error" + }, + "503": { + "description": "Service too busy" + }, + "default": { + "description": "Unexpected error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + }, + "post": { + "summary": "modify schema or table permissions", + "tags": [ + "Custom Features" + ], + "description": "Modify any of the public flag, readGroup or writeGroup of a schema or table. These operations are permitted by the owner of the schema and owners of the target tables. Parameters are in-line CFLF separated entries in key=value format. An example of a groupURI is ivo://cadc.nrc.ca/gms?myGroup\n", + "parameters": [ + { + "name": "name", + "in": "path", + "description": "name of the schema or table to modify.", + "required": true, + "type": "string" + }, + { + "name": "public", + "in": "body", + "description": "public=true or public=false", + "required": false, + "type": "string" + }, + { + "name": "r-group", + "in": "body", + "description": "set or clear read group. r-group=groupURI or r-group= to clear. group URI in format ivo://cadc.nrc.ca/gms?groupName", + "required": false, + "type": "string" + }, + { + "name": "rw-group", + "in": "body", + "description": "set or clear read/write group. rw-group=groupURI or rw-group= to clear. group URI in format ivo://cadc.nrc.ca/gms?groupName", + "required": false, + "type": "string" + } + ], + "consumes": [ + "text/plain" + ], + "responses": { + "200": { + "description": "permissions updated successfully" + }, + "400": { + "description": "invalid permission syntax" + }, + "403": { + "description": "Insufficient permission to change permissions" + }, + "500": { + "description": "Internal server error" + }, + "503": { + "description": "Service too busy" + }, + "default": { + "description": "Unexpected error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + } + }, + "/load/{tableName}": { + "post": { + "summary": "sync streaming row insert", + "tags": [ + "Custom Features" + ], + "description": "Bulk load rows and append to the table. This currently supports input data in CSV and TSV format (with header), and FITS Binary Tables\n", + "parameters": [ + { + "name": "tableName", + "in": "path", + "description": "a single fully qualified table name (e.g. tap_schema.tables)", + "required": true, + "type": "string" + } + ], + "consumes": [ + "text/csv", + "text/tab-separated-values", + "application/fits" + ], + "responses": { + "200": { + "description": "rows successfully inserted", + "schema": { + "$ref": "#/definitions/tables" + } + }, + "400": { + "description": "insert rows failed due to invalid input" + }, + "500": { + "description": "Internal server error" + }, + "503": { + "description": "Service too busy" + }, + "default": { + "description": "Unexpected error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + } + }, + "/table-update": { + "post": { + "summary": "async table update (create UWS Job)", + "tags": [ + "Custom Features" + ], + "description": "Asynchronous table update currently update supports index creation. In future it will also support loading content (rows) from a remote location (URI)", + "parameters": [ + { + "name": "TABLE", + "in": "query", + "description": "the table to update (TABLE={table name}): fully-qualified name of the table to update", + "required": true, + "type": "string" + }, + { + "name": "INDEX", + "in": "query", + "description": "index a column (INDEX={column name}): value is the name of the column to index; currently limited to single column only", + "required": true, + "type": "string" + }, + { + "name": "UNIQUE", + "in": "query", + "description": "create unique index (UNIQUE=true): use with INDEX={column name} to make a unique index (default: false)", + "required": false, + "type": "boolean" + } + ], + "produces": [ + "text/xml" + ], + "responses": { + "200": { + "description": "Successful response" + }, + "303": { + "description": "standard UWS redirect to the created job" + }, + "401": { + "description": "Unauthorized - User not authenticated" + }, + "404": { + "description": "Not Found - User not found" + }, + "500": { + "description": "Internal error" + }, + "503": { + "description": "Service busy" + }, + "default": { + "description": "Unexpeced error", + "schema": { + "$ref": "#/definitions/Error" + } + } + } + } + } + }, + "definitions": { + "Job": { + "type": "string" + }, + "availability": { + "type": "object", + "xml": { + "name": "availability", + "namespace": "http://www.ivoa.net/xml/VOSIAvailability/v1.0", + "prefix": "vosi" + }, + "properties": { + "available": { + "type": "boolean", + "xml": { + "attribute": true, + "prefix": "vosi" + } + }, + "note": { + "type": "string", + "xml": { + "attribute": true, + "prefix": "vosi" + } + } + } + }, + "tables": { + "type": "object", + "xml": { + "namespace": "http://www.ivoa.net/xml/VOSITables/v1.0", + "prefix": "vosi" + }, + "properties": { + "available": { + "type": "boolean", + "xml": { + "attribute": true, + "prefix": "vosi" + } + }, + "note": { + "type": "string", + "xml": { + "attribute": true, + "prefix": "vosi" + } + } + } + }, + "capabilities": { + "type": "object", + "xml": { + "namespace": "http://www.ivoa.net/xml/VOSICapabilities/v1.0", + "prefix": "vosi" + } + }, + "Error": { + "required": [ + "code", + "message" + ], + "properties": { + "code": { + "type": "integer", + "format": "int32" + }, + "message": { + "type": "string" + } + } + } + } +} diff --git a/youcat/src/test/java/org/opencadc/youcat/TestDataSource.java b/youcat/src/test/java/org/opencadc/youcat/TestDataSource.java new file mode 100644 index 00000000..89e14861 --- /dev/null +++ b/youcat/src/test/java/org/opencadc/youcat/TestDataSource.java @@ -0,0 +1,70 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2009. (c) 2009. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* $Revision: 1 $ +* +************************************************************************ +*/ + +package org.opencadc.youcat; + +import ca.nrc.cadc.db.ConnectionConfig; +import ca.nrc.cadc.db.DBConfig; +import ca.nrc.cadc.db.DBUtil; +import javax.sql.DataSource; + +import org.apache.log4j.Logger; + +public class TestDataSource +{ + private static final Logger LOG = Logger.getLogger(TestDataSource.class); + + public static DataSource getDataSource(String server, String database) + { + try + { + DBConfig dbConfig = new DBConfig(); + ConnectionConfig connectionConfig = dbConfig.getConnectionConfig(server, database); + DataSource ds = DBUtil.getDataSource(connectionConfig, true, true); + return ds; + } + catch (Exception ex) + { + LOG.error(ex); + } + + return null; + } + +} diff --git a/youcat/src/test/java/org/opencadc/youcat/tap/AdqlQueryImplTest.java b/youcat/src/test/java/org/opencadc/youcat/tap/AdqlQueryImplTest.java new file mode 100644 index 00000000..8b17e1b3 --- /dev/null +++ b/youcat/src/test/java/org/opencadc/youcat/tap/AdqlQueryImplTest.java @@ -0,0 +1,208 @@ +/* +************************************************************************ +******************* CANADIAN ASTRONOMY DATA CENTRE ******************* +************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** +* +* (c) 2014. (c) 2014. +* Government of Canada Gouvernement du Canada +* National Research Council Conseil national de recherches +* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 +* All rights reserved Tous droits réservés +* +* NRC disclaims any warranties, Le CNRC dénie toute garantie +* expressed, implied, or énoncée, implicite ou légale, +* statutory, of any kind with de quelque nature que ce +* respect to the software, soit, concernant le logiciel, +* including without limitation y compris sans restriction +* any warranty of merchantability toute garantie de valeur +* or fitness for a particular marchande ou de pertinence +* purpose. NRC shall not be pour un usage particulier. +* liable in any event for any Le CNRC ne pourra en aucun cas +* damages, whether direct or être tenu responsable de tout +* indirect, special or general, dommage, direct ou indirect, +* consequential or incidental, particulier ou général, +* arising from the use of the accessoire ou fortuit, résultant +* software. Neither the name de l'utilisation du logiciel. Ni +* of the National Research le nom du Conseil National de +* Council of Canada nor the Recherches du Canada ni les noms +* names of its contributors may de ses participants ne peuvent +* be used to endorse or promote être utilisés pour approuver ou +* products derived from this promouvoir les produits dérivés +* software without specific prior de ce logiciel sans autorisation +* written permission. préalable et particulière +* par écrit. +* +* This file is part of the Ce fichier fait partie du projet +* OpenCADC project. OpenCADC. +* +* OpenCADC is free software: OpenCADC est un logiciel libre ; +* you can redistribute it and/or vous pouvez le redistribuer ou le +* modify it under the terms of modifier suivant les termes de +* the GNU Affero General Public la “GNU Affero General Public +* License as published by the License” telle que publiée +* Free Software Foundation, par la Free Software Foundation +* either version 3 of the : soit la version 3 de cette +* License, or (at your option) licence, soit (à votre gré) +* any later version. toute version ultérieure. +* +* OpenCADC is distributed in the OpenCADC est distribué +* hope that it will be useful, dans l’espoir qu’il vous +* but WITHOUT ANY WARRANTY; sera utile, mais SANS AUCUNE +* without even the implied GARANTIE : sans même la garantie +* warranty of MERCHANTABILITY implicite de COMMERCIALISABILITÉ +* or FITNESS FOR A PARTICULAR ni d’ADÉQUATION À UN OBJECTIF +* PURPOSE. See the GNU Affero PARTICULIER. Consultez la Licence +* General Public License for Générale Publique GNU Affero +* more details. pour plus de détails. +* +* You should have received Vous devriez avoir reçu une +* a copy of the GNU Affero copie de la Licence Générale +* General Public License along Publique GNU Affero avec +* with OpenCADC. If not, see OpenCADC ; si ce n’est +* . pas le cas, consultez : +* . +* +* $Revision: 5 $ +* +************************************************************************ + */ + +package org.opencadc.youcat.tap; + +import org.opencadc.youcat.tap.AdqlQueryImpl; +import ca.nrc.cadc.tap.AdqlQuery; +import ca.nrc.cadc.tap.schema.ColumnDesc; +import ca.nrc.cadc.tap.schema.FunctionDesc; +import ca.nrc.cadc.tap.schema.SchemaDesc; +import ca.nrc.cadc.tap.schema.TableDesc; +import ca.nrc.cadc.tap.schema.TapDataType; +import ca.nrc.cadc.tap.schema.TapSchema; +import ca.nrc.cadc.util.Log4jInit; +import ca.nrc.cadc.util.PropertiesReader; +import ca.nrc.cadc.uws.Job; +import ca.nrc.cadc.uws.Parameter; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; + +/** + * + * @author pdowler + */ +public class AdqlQueryImplTest { + + private static final Logger log = Logger.getLogger(AdqlQueryImplTest.class); + + static { + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.tap", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.reg", Level.DEBUG); + Log4jInit.setLevel("ca.nrc.cadc.util", Level.DEBUG); + } + + public AdqlQueryImplTest() { + } + + @Test + public void testTOP() { + System.setProperty(PropertiesReader.CONFIG_DIR_SYSTEM_PROPERTY, "build/resources/test/config"); + try { + String adql = "select TOP 5 foo from test.SomeTable"; + + Job job = new Job() { + public String getID() { + return "testJob"; + } + }; + job.getParameterList().add(new Parameter("QUERY", adql)); + + TapSchema tapSchema = loadTapSchema(); + + AdqlQuery q = new AdqlQueryImpl(); + q.setJob(job); + q.setTapSchema(tapSchema); + + String sql = q.getSQL(); + Assert.assertNotNull(sql); + log.info("testTOP SQL: " + sql); + sql = sql.toLowerCase(); + + int i = sql.indexOf("limit"); + Assert.assertTrue("found limit", (i > 0)); + + String limit = sql.substring(i).trim(); + log.info("testTOP: " + limit); + + Assert.assertEquals("limit 5", limit); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } finally { + System.clearProperty(PropertiesReader.CONFIG_DIR_SYSTEM_PROPERTY); + } + } + + @Test + public void testRegionConverterEnabled() { + System.setProperty(PropertiesReader.CONFIG_DIR_SYSTEM_PROPERTY, "build/resources/test/config"); + try { + String adql = "select * from test.SomeTable" + + " where CONTAINS(pos, CIRCLE(NULL, 12.0, 34.0, 0.2)) = 1"; + + Job job = new Job() { + public String getID() { + return "testJob"; + } + }; + job.getParameterList().add(new Parameter("QUERY", adql)); + + TapSchema tapSchema = loadTapSchema(); + + AdqlQuery q = new AdqlQueryImpl(); + q.setJob(job); + q.setTapSchema(tapSchema); + + String sql = q.getSQL(); + Assert.assertNotNull(sql); + log.info("testRegionConverterEnabled SQL: " + sql); + sql = sql.toLowerCase(); + + int i = sql.indexOf("where"); + Assert.assertTrue("found where", (i > 0)); + + String where = sql.substring(i); + log.info("testRegionConverterEnabled: " + where); + + Assert.assertTrue(where.contains("pos <@ scircle(spoint(radians(12.0), radians(34.0)), radians(0.2))")); + } catch (Exception unexpected) { + log.error("unexpected exception", unexpected); + Assert.fail("unexpected exception: " + unexpected); + } finally { + System.clearProperty(PropertiesReader.CONFIG_DIR_SYSTEM_PROPERTY); + } + } + + private TapSchema loadTapSchema() { + TapSchema ret = new TapSchema(); + + SchemaDesc sd = new SchemaDesc("test"); + TableDesc td = new TableDesc("test", "test.SomeTable"); + td.getColumnDescs().add(new ColumnDesc(td.getTableName(), "foo", TapDataType.STRING)); + td.getColumnDescs().add(new ColumnDesc(td.getTableName(), "pos", TapDataType.POINT)); + sd.getTableDescs().add(td); + ret.getSchemaDescs().add(sd); + + ret.getFunctionDescs().add(new FunctionDesc("interval", TapDataType.INTERVAL)); + ret.getFunctionDescs().add(new FunctionDesc("point", TapDataType.POINT)); + ret.getFunctionDescs().add(new FunctionDesc("circle", TapDataType.CIRCLE)); + ret.getFunctionDescs().add(new FunctionDesc("polygon", TapDataType.POLYGON)); + + ret.getFunctionDescs().add(new FunctionDesc("contains", TapDataType.INTEGER)); + ret.getFunctionDescs().add(new FunctionDesc("intersects", TapDataType.INTEGER)); + ret.getFunctionDescs().add(new FunctionDesc("coord1", TapDataType.DOUBLE)); + ret.getFunctionDescs().add(new FunctionDesc("coord2", TapDataType.DOUBLE)); + + return ret; + } +} diff --git a/youcat/src/test/java/org/opencadc/youcat/tap/MaxRecValidatorImplTest.java b/youcat/src/test/java/org/opencadc/youcat/tap/MaxRecValidatorImplTest.java new file mode 100644 index 00000000..20c16e34 --- /dev/null +++ b/youcat/src/test/java/org/opencadc/youcat/tap/MaxRecValidatorImplTest.java @@ -0,0 +1,160 @@ +/* + ************************************************************************ + **** C A N A D I A N A S T R O N O M Y D A T A C E N T R E ***** + * + * (c) 2012. (c) 2012. + * National Research Council Conseil national de recherches + * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 + * All rights reserved Tous droits reserves + * + * NRC disclaims any warranties Le CNRC denie toute garantie + * expressed, implied, or statu- enoncee, implicite ou legale, + * tory, of any kind with respect de quelque nature que se soit, + * to the software, including concernant le logiciel, y com- + * without limitation any war- pris sans restriction toute + * ranty of merchantability or garantie de valeur marchande + * fitness for a particular pur- ou de pertinence pour un usage + * pose. NRC shall not be liable particulier. Le CNRC ne + * in any event for any damages, pourra en aucun cas etre tenu + * whether direct or indirect, responsable de tout dommage, + * special or general, consequen- direct ou indirect, particul- + * tial or incidental, arising ier ou general, accessoire ou + * from the use of the software. fortuit, resultant de l'utili- + * sation du logiciel. + * + * + * @author jenkinsd + * 7/3/12 - 2:45 PM + * + * + * + **** C A N A D I A N A S T R O N O M Y D A T A C E N T R E ***** + ************************************************************************ + */ +package org.opencadc.youcat.tap; + +import org.opencadc.youcat.tap.MaxRecValidatorImpl; +import ca.nrc.cadc.tap.MaxRecValidator; +import ca.nrc.cadc.tap.schema.ColumnDesc; +import ca.nrc.cadc.tap.schema.SchemaDesc; +import ca.nrc.cadc.tap.schema.TableDesc; +import ca.nrc.cadc.tap.schema.TapDataType; +import ca.nrc.cadc.tap.schema.TapSchema; +import ca.nrc.cadc.util.Log4jInit; +import ca.nrc.cadc.uws.Job; +import ca.nrc.cadc.uws.Parameter; +import java.util.ArrayList; +import java.util.List; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import static org.junit.Assert.*; +import org.junit.Test; + +public class MaxRecValidatorImplTest +{ + private static final Logger log = Logger.getLogger(MaxRecValidatorImplTest.class); + + static final TapSchema tapSchema = new TapSchema(); + + static + { + Log4jInit.setLevel("ca.nrc.cadc.cat", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.tap", Level.INFO); + Log4jInit.setLevel("ca.nrc.cadc.dali", Level.INFO); + + + ColumnDesc c1 = new ColumnDesc("foo.bar", "a", TapDataType.INTEGER); + ColumnDesc c2 = new ColumnDesc("foo.bar", "b", TapDataType.DOUBLE); + ColumnDesc c3 = new ColumnDesc("foo.bar", "c", new TapDataType("char", "64*", null)); + + TableDesc td = new TableDesc("foo", "foo.bar"); + td.getColumnDescs().add(c1); + td.getColumnDescs().add(c2); + td.getColumnDescs().add(c3); + + SchemaDesc sd = new SchemaDesc("foo"); + sd.getTableDescs().add(td); + + tapSchema.getSchemaDescs().add(sd); + } + + + + Job job = new Job() + { + @Override + public String getID() { return "internal-test-jobID"; } + }; + + @Test + public void testSync() throws Exception + { + try + { + log.debug("*** testSync ***"); + job.getParameterList().add(new Parameter("QUERY", "SELECT a, b, c FROM foo.bar")); + + final MaxRecValidator testSubject = new MaxRecValidatorImpl(); + testSubject.setJob(job); + testSubject.setTapSchema(tapSchema); + testSubject.setSynchronousMode(true); + + // no limit + final Integer result1 = testSubject.validate(); + log.debug("no limit: " + result1); + assertNull("no limit", result1); + + // large user limit + job.getParameterList().add(new Parameter("MAXREC", "123456")); + final int result2 = testSubject.validate(); + assertEquals("user limit", 123456, result2); + + // sync -> vospace: user limit only + job.getParameterList().add(new Parameter("DEST", "vos://cadc.nrc.ca!vospace/myvospace")); + final int result3 = testSubject.validate(); + assertEquals("vospace dest with user limit", 123456, result3); + } + finally + { + job.getParameterList().clear(); + } + } + + @Test + public void testASync() throws Exception + { + try + { + log.debug("*** testASync ***"); + job.getParameterList().add(new Parameter("QUERY", "SELECT a, b, c FROM foo.bar")); + + final MaxRecValidator testSubject = new MaxRecValidatorImpl(); + testSubject.setJob(job); + testSubject.setTapSchema(tapSchema); + testSubject.setSynchronousMode(false); + + // imposed limit + final Integer defaultLimit = testSubject.validate(); + log.debug("async limit: " + defaultLimit); + assertNotNull("async limit", defaultLimit); + + // large user limit + Integer largeLimit = defaultLimit * 100; + job.getParameterList().add(new Parameter("MAXREC", largeLimit.toString())); + final Integer result2 = testSubject.validate(); + log.debug("large limit: " + defaultLimit); + assertNotNull("async limit", result2); + assertEquals("larger == dynamic", defaultLimit, result2); + + // sync -> vospace: user limit only + job.getParameterList().add(new Parameter("DEST", "vos://cadc.nrc.ca!vospace/myvospace")); + final Integer result3 = testSubject.validate(); + assertNotNull("user limit", result3); + assertEquals("vospace dest with user limit", largeLimit, result3); + } + finally + { + job.getParameterList().clear(); + } + } +} diff --git a/youcat/src/test/resources/config/cadc-registry.properties b/youcat/src/test/resources/config/cadc-registry.properties new file mode 100644 index 00000000..ebe80266 --- /dev/null +++ b/youcat/src/test/resources/config/cadc-registry.properties @@ -0,0 +1,11 @@ +# +# local authority map to find the service that provides +# the local implementation of an API (standardID) +# +# configure RegistryClient bootstrap +ca.nrc.cadc.reg.client.RegistryClient.baseURL = https://ws.cadc-ccda.hia-iha.nrc-cnrc.gc.ca/reg + +# configure LocalAuthority lookups +# = +ivo://ivoa.net/std/GMS#search-1.0 = ivo://cadc.nrc.ca/gms +ivo://ivoa.net/std/GMS#search-0.1 = ivo://cadc.nrc.ca/gms From fab15013c8abbd35f80725cf7ed04863e6543718 Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Tue, 25 Apr 2023 20:55:31 -0700 Subject: [PATCH 02/23] youcat: build tweak --- youcat/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/youcat/build.gradle b/youcat/build.gradle index 5ab98d0a..1990fa7d 100644 --- a/youcat/build.gradle +++ b/youcat/build.gradle @@ -42,9 +42,9 @@ dependencies { compile 'org.opencadc:cadc-uws-server:[1.2.6,)' compile 'org.opencadc:cadc-tap-tmp:[1.1,)' compile 'org.opencadc:cadc-gms:[1.0,)' - runtime 'org.opencadc:cadc-registry:[1.5.10,)' + compile 'org.opencadc:cadc-registry:[1.5.10,)' + compile 'org.opencadc:cadc-gms:[1.0.4,2.0)' - runtime 'org.opencadc:cadc-gms:[1.0.4,2.0)' runtime 'org.opencadc:cadc-access-control-identity:[1.1.0,)' runtime 'org.apache.ant:ant:1.9.4' runtime 'org.opencadc:nom-tam-fits:[1.16.0,2.0)' From 28b8d6f5fa6801d5e4d4b6887a622e21fd9d42df Mon Sep 17 00:00:00 2001 From: Patrick Dowler Date: Wed, 26 Apr 2023 09:48:46 -0700 Subject: [PATCH 03/23] youcat: remove obsolete auth tests, bug fixes and README improvements after intTest --- youcat/README.md | 42 +++-- youcat/build.gradle | 1 + .../youcat/RegistryClientLookupTest.java | 71 +-------- .../opencadc/youcat/CatalogTapService.java | 2 +- youcat/src/main/webapp/META-INF/context.xml | 18 +-- youcat/src/main/webapp/WEB-INF/web.xml | 147 +++++------------- youcat/src/main/webapp/index.html | 4 +- 7 files changed, 88 insertions(+), 197 deletions(-) diff --git a/youcat/README.md b/youcat/README.md index b476a5cd..e449798c 100644 --- a/youcat/README.md +++ b/youcat/README.md @@ -1,8 +1,14 @@ -# youcat +# table query service with user-managed tables (youcat) -YouCat is a complete TAP service implementation that supports TAP-1.1 plus -CADC/CANFAR extensions to enable users to create their own tables and load -data into them. +This service is a complete TAP service implementation that supports the +IVOA TAP-1.1 web +service API plus CADC/CANFAR extensions to enable users to create their own +tables and load data into them. + +The extended features and current implementation are aimed primarily at +creating and bulk-loading of astronomical catalogues, but it can be used +as a generic TAP implementation(see configuration limitations, development +plans, and enahncements needed below). ## configuration @@ -19,15 +25,26 @@ for system properties related to the deployment environment. See cadc-util for common system properties. -`baldur` includes multiple IdentityManager implementations to support authenticated access: +`youcat` includes multiple IdentityManager implementations to support authenticated access: - See cadc-access-control-identity for CADC access-control system support. - See cadc-gms for OIDC token support. ### cadc-registry.properties +The `youcat` service currently queries a known (trusted) GMS service for all groups a user belongs to +in order to mangle queries on the *tap_schema* content. A service providing the +`ivo://ivoa.net/std/GMS#search-1.0` capability must be configured here for that to work. + See cadc-registry. +### cadc-tap-tmp.properties + +This service uses the `cadc-tap-tmp` library to persist tap_upload tables and async results. It is currently +hard-coded (PluginFactory.properties) to use the HttpStorageManager and persist to an external URL (HTTP PUT). + +See cadc-tap-tmp. + ## youcat.properties As hard-coded behaviours of `youcat` are extracted from the build and made configurable, @@ -40,19 +57,20 @@ The `youcat` implementation is currently hard-coded to use a PostgreSQL backend Make all aspects of the service configurable at runtime so deployers can use the standard image published by CADC: -- move some code from youcat into appropriate opencadc library - -- extract PluginFactory.properties from inside war (cadc-tap-server) - - document database requirements and configuration - -- include support for standard token authentication (no local code needed) +- document user allocation process (schema creation and addition to *tap_schema*), maybe provide tools +- move some code from youcat into appropriate opencadc library +- extract PluginFactory.properties from inside war to enable more configuration at runtime (cadc-tap-server) +- document integration test requirements (currently very CADC-specific and not usable by external developers) ## enhancements Add *tap_schema* content management and configuration for non-user tables. -Add configuration to disable user-tables so deployed service is a normal TAP service. +Add support for the *tap_schema* to be in a different database from the content. + +Add configuration option to disable user-tables so deployed service is a normal TAP service? This might just be +a separate build/service/image... TBD. diff --git a/youcat/build.gradle b/youcat/build.gradle index 1990fa7d..c4ee2139 100644 --- a/youcat/build.gradle +++ b/youcat/build.gradle @@ -45,6 +45,7 @@ dependencies { compile 'org.opencadc:cadc-registry:[1.5.10,)' compile 'org.opencadc:cadc-gms:[1.0.4,2.0)' + runtime 'org.opencadc:cadc-vos:[1.1,2.0)' runtime 'org.opencadc:cadc-access-control-identity:[1.1.0,)' runtime 'org.apache.ant:ant:1.9.4' runtime 'org.opencadc:nom-tam-fits:[1.16.0,2.0)' diff --git a/youcat/src/intTest/java/org/opencadc/youcat/RegistryClientLookupTest.java b/youcat/src/intTest/java/org/opencadc/youcat/RegistryClientLookupTest.java index b6dc1a9f..f1260d4f 100644 --- a/youcat/src/intTest/java/org/opencadc/youcat/RegistryClientLookupTest.java +++ b/youcat/src/intTest/java/org/opencadc/youcat/RegistryClientLookupTest.java @@ -3,7 +3,7 @@ ******************* CANADIAN ASTRONOMY DATA CENTRE ******************* ************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES ************** * -* (c) 2019. (c) 2019. +* (c) 2023. (c) 2023. * Government of Canada Gouvernement du Canada * National Research Council Conseil national de recherches * Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6 @@ -80,7 +80,6 @@ import ca.nrc.cadc.reg.client.RegistryClient; import ca.nrc.cadc.util.FileUtil; import ca.nrc.cadc.util.Log4jInit; - import java.io.ByteArrayOutputStream; import java.io.File; import java.net.URL; @@ -89,9 +88,7 @@ import java.nio.file.Paths; import java.util.Map; import java.util.TreeMap; - import javax.security.auth.Subject; - import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.xerces.impl.dv.util.Base64; @@ -276,71 +273,7 @@ public void testX509Tables() { } } - // username-password endpoints exist by convention but are not described in the - // VOSI-capabilities - @Test - public void testAuthAsync() { - try { - URL url = tapClient.getAsyncURL(Standards.SECURITY_METHOD_ANON); - url = new URL(url.toExternalForm().replace("async", "auth-async")); - - Assert.assertNotNull(url); - HttpPost post = new HttpPost(url, queryParams, false); - post.setRequestProperty(AuthenticationUtil.AUTHORIZATION_HEADER, basicAuth); - post.run(); - Assert.assertNull(post.getThrowable()); - Assert.assertEquals(303, post.getResponseCode()); - Assert.assertNotNull(post.getRedirectURL()); - Assert.assertEquals(userid, post.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); - } catch (Exception unexpected) { - log.error("unexpected exception", unexpected); - Assert.fail("unexpected exception: " + unexpected); - } - } - - @Test - public void testAuthSync() { - try { - URL url = tapClient.getSyncURL(Standards.SECURITY_METHOD_ANON); - url = new URL(url.toExternalForm().replace("sync", "auth-sync")); - - Assert.assertNotNull(url); - log.info("url: " + url.toString()); - HttpPost post = new HttpPost(url, queryParams, false); - post.setRequestProperty(AuthenticationUtil.AUTHORIZATION_HEADER, basicAuth); - post.run(); - Assert.assertNull(post.getThrowable()); - Assert.assertEquals(303, post.getResponseCode()); - Assert.assertNotNull(post.getRedirectURL()); - Assert.assertEquals(userid, post.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); - } catch (Exception unexpected) { - log.error("unexpected exception", unexpected); - Assert.fail("unexpected exception: " + unexpected); - } - } - - @Test - public void testAuthTables() { - try { - URL url = regClient.getServiceURL(Constants.RESOURCE_ID, Standards.VOSI_TABLES_11, AuthMethod.ANON); - url = new URL(url.toExternalForm().replace("tables", "auth-tables")); - - Assert.assertNotNull(url); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - HttpDownload get = new HttpDownload(url, bos); - get.setRequestProperty(AuthenticationUtil.AUTHORIZATION_HEADER, basicAuth); - get.run(); - Assert.assertNull(get.getThrowable()); - Assert.assertEquals(200, get.getResponseCode()); - Assert.assertTrue(bos.size() > 0); - Assert.assertEquals(userid, get.getResponseHeader(AuthenticationUtil.VO_AUTHENTICATED_HEADER)); - - } catch (Exception unexpected) { - log.error("unexpected exception", unexpected); - Assert.fail("unexpected exception: " + unexpected); - } - } - + // internal log control @Test public void testLogControl() { diff --git a/youcat/src/main/java/org/opencadc/youcat/CatalogTapService.java b/youcat/src/main/java/org/opencadc/youcat/CatalogTapService.java index ec46d035..43d8a2da 100644 --- a/youcat/src/main/java/org/opencadc/youcat/CatalogTapService.java +++ b/youcat/src/main/java/org/opencadc/youcat/CatalogTapService.java @@ -169,7 +169,7 @@ public Availability getStatus() { checkResource = new CheckWebService(url); checkResource.check(); - URI groupsURI = localAuthority.getServiceURI(Standards.GMS_SEARCH_01.toString()); + URI groupsURI = localAuthority.getServiceURI(Standards.GMS_SEARCH_10.toString()); if (!groupsURI.equals(usersURI)) { url = reg.getServiceURL(groupsURI, Standards.VOSI_AVAILABILITY, AuthMethod.ANON); checkResource = new CheckWebService(url); diff --git a/youcat/src/main/webapp/META-INF/context.xml b/youcat/src/main/webapp/META-INF/context.xml index 2c1bb428..ddc33c4f 100644 --- a/youcat/src/main/webapp/META-INF/context.xml +++ b/youcat/src/main/webapp/META-INF/context.xml @@ -10,9 +10,9 @@ factory="org.apache.tomcat.jdbc.pool.DataSourceFactory" closeMethod="close" minEvictableIdleTimeMillis="60000" timeBetweenEvictionRunsMillis="30000" maxWait="20000" - initialSize="0" minIdle="0" maxIdle="${youcat.tapadm.maxActive}" maxActive="${youcat.tapadm.maxActive}" - username="${youcat.tapadm.username}" password="${youcat.tapadm.password}" - driverClassName="org.postgresql.Driver" url="${youcat.tapadm.url}" + initialSize="0" minIdle="0" maxIdle="${org.opencadc.youcat.tapadm.maxActive}" maxActive="${org.opencadc.youcat.tapadm.maxActive}" + username="${org.opencadc.youcat.tapadm.username}" password="${org.opencadc.youcat.tapadm.password}" + driverClassName="org.postgresql.Driver" url="${org.opencadc.youcat.tapadm.url}" removeAbandoned="false" testWhileIdle="true" testOnBorrow="true" validationQuery="select 123" /> diff --git a/youcat/src/main/webapp/WEB-INF/web.xml b/youcat/src/main/webapp/WEB-INF/web.xml index 2a5bd68c..5ee262a9 100644 --- a/youcat/src/main/webapp/WEB-INF/web.xml +++ b/youcat/src/main/webapp/WEB-INF/web.xml @@ -19,6 +19,7 @@ logLevelPackages + org.opencadc.youcat org.opencadc.tap ca.nrc.cadc.rest ca.nrc.cadc.uws @@ -30,14 +31,6 @@ ca.nrc.cadc.auth ca.nrc.cadc.vosi - - - logAccessGroup - ivo://cadc.nrc.ca/gms?CADC - - - groupAuthorizer - ca.nrc.cadc.ac.client.GroupAuthorizer logControlProperties @@ -51,7 +44,7 @@ ca.nrc.cadc.uws.server.JobServlet ca.nrc.cadc.uws.server.JobManager - ca.nrc.cadc.cat.ws.QueryJobManager + org.opencadc.youcat.QueryJobManager ca.nrc.cadc.rest.InlineContentHandler @@ -77,7 +70,7 @@ ca.nrc.cadc.uws.server.JobServlet ca.nrc.cadc.uws.server.JobManager - ca.nrc.cadc.cat.ws.QueryJobManager + org.opencadc.youcat.QueryJobManager ca.nrc.cadc.rest.InlineContentHandler @@ -123,7 +116,7 @@ ca.nrc.cadc.vosi.AvailabilityPlugin - ca.nrc.cadc.cat.ws.CatalogTapService + org.opencadc.youcat.CatalogTapService availabilityProperties @@ -161,7 +154,7 @@ ca.nrc.cadc.uws.server.JobServlet ca.nrc.cadc.uws.server.JobManager - ca.nrc.cadc.cat.ws.TableUpdateJobManager + org.opencadc.youcat.TableUpdateJobManager get @@ -202,97 +195,43 @@ 3 - - AsyncQueryServlet - /async/* - - - AsyncQueryServlet - /auth-async/* - - - SyncQueryServlet - /sync/* - - - SyncQueryServlet - /auth-sync/* - - - TableServlet - /tables/* - - - TableServlet - /auth-tables/* - - - AsyncTableUpdateServlet - /table-update/* - - - AsyncTableUpdateServlet - /auth-table-update/* - - - PermissionsServlet - /permissions/* - - - PermissionsServlet - /auth-permissions/* - - - SyncLoadServlet - /load/* - - - SyncLoadServlet - /auth-load/* - - - - - CapabilitiesServlet - /capabilities - - - logControl - /logControl/* - - - - AvailabilityServlet - /availability - - - - - - auth - - /auth-async/* - /auth-sync/* - /auth-tables/* - /auth-table-update/* - /auth-load/* - GET - POST - HEAD - PUT - OPTIONS - TRACE - DELETE - - - force authentication for all requests - public - - - - - BASIC - Canadian Astronomy Data Centre - + + AsyncQueryServlet + /async/* + + + SyncQueryServlet + /sync/* + + + TableServlet + /tables/* + + + AsyncTableUpdateServlet + /table-update/* + + + PermissionsServlet + /permissions/* + + + SyncLoadServlet + /load/* + + + + AvailabilityServlet + /availability + + + CapabilitiesServlet + /capabilities + + + logControl + /logControl/* + + diff --git a/youcat/src/main/webapp/index.html b/youcat/src/main/webapp/index.html index 82bfb75b..67b991f8 100644 --- a/youcat/src/main/webapp/index.html +++ b/youcat/src/main/webapp/index.html @@ -2,7 +2,7 @@ - Swagger UI + youcat API