#288: fixed integer mapping for BigQueryQueryRewriter (#312)

* #288: Fixed integer mapping for BigQueryQueryRewriter, improved other types mapping, updated BigQuery documentation, fixed bug with an empty table, improved tests
exasol · Feb 13, 2020 · a5aeb00 · a5aeb00
1 parent 34e2a49
commit a5aeb00
Show file tree

Hide file tree

Showing 24 changed files with 226 additions and 152 deletions.
diff --git a/doc/dialects/athena.md b/doc/dialects/athena.md
@@ -49,7 +49,7 @@ The SQL statement below creates the adapter script, defines the Java class that
 ```sql
 CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
     %scriptclass com.exasol.adapter.RequestDispatcher;
-    %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+    %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.2.jar;
     %jar /buckets/<BFS service>/<bucket>/AthenaJDBC42-<JDBC driver version>.jar;
 /
 ;

diff --git a/doc/dialects/aurora.md b/doc/dialects/aurora.md
@@ -62,7 +62,7 @@ The SQL statement below creates the adapter script, defines the Java class that
 ```sql
 CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
     %scriptclass com.exasol.adapter.RequestDispatcher;
-    %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+    %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.2.jar;
     %jar /buckets/<BFS service>/<bucket>/postgresql-<JDBC driver version>.jar;
 /
 ```

diff --git a/doc/dialects/bigquery.md b/doc/dialects/bigquery.md
@@ -11,35 +11,39 @@ Download the [Simba JDBC Driver for Google BigQuery](https://cloud.google.com/bi
 1. [Create a bucket in BucketFS](https://docs.exasol.com/administration/on-premise/bucketfs/create_new_bucket_in_bucketfs_service.htm) 
 1. Upload the driver to BucketFS
 
+Hint: Magnitude Simba driver contains a lot of jar files, but you can upload all of them together as an archive (`.tar.gz`, for example).
+The archive will be unpacked automatically in the bucket and you can access the files using the following path pattern '<your bucket>/<archive's name without extension>/<name of a file form the archive>.jar'
+
 ## Installing the Adapter Script
 
 Upload the latest available release of [Virtual Schema JDBC Adapter](https://github.com/exasol/virtual-schemas/releases) to Bucket FS.
 
 Then create a schema to hold the adapter script.
 
 ```sql
-CREATE SCHEMA ADAPTER;
+CREATE SCHEMA SCHEMA_FOR_VS_SCRIPT;
 ```
 
 The SQL statement below creates the adapter script, defines the Java class that serves as entry point and tells the UDF framework where to find the libraries (JAR files) for Virtual Schema and database driver.
 
 Please remember to check the versions of your JAR files after downloading driver. They can differ from the list below.
 
 ```sql
-CREATE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
+CREATE JAVA ADAPTER SCRIPT SCHEMA_FOR_VS_SCRIPT.ADAPTER_SCRIPT_BIGQUERY AS
     %scriptclass com.exasol.adapter.RequestDispatcher;
-    %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+    %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.2.jar;
     %jar /buckets/<BFS service>/<bucket>/avro-1.8.2.jar;
-    %jar /buckets/<BFS service>/<bucket>/gax-1.40.0.jar;
+    %jar /buckets/<BFS service>/<bucket>/gax-1.42.0.jar;
     %jar /buckets/<BFS service>/<bucket>/google-api-client-1.28.0.jar;
     %jar /buckets/<BFS service>/<bucket>/google-api-services-bigquery-v2-rev426-1.25.0.jar;
-    %jar /buckets/<BFS service>/<bucket>/google-auth-library-credentials-0.13.0.jar;
+    %jar /buckets/<BFS service>/<bucket>/google-auth-library-credentials-0.15.0.jar;
     %jar /buckets/<BFS service>/<bucket>/google-auth-library-oauth2-http-0.13.0.jar;
     %jar /buckets/<BFS service>/<bucket>/GoogleBigQueryJDBC42.jar;
-    %jar /buckets/<BFS service>/<bucket>/google-http-client-1.28.0.jar;
+    %jar /buckets/<BFS service>/<bucket>/google-http-client-1.29.0.jar;
     %jar /buckets/<BFS service>/<bucket>/google-http-client-jackson2-1.28.0.jar;
     %jar /buckets/<BFS service>/<bucket>/google-oauth-client-1.28.0.jar;
     %jar /buckets/<BFS service>/<bucket>/grpc-context-1.18.0.jar;
+    %jar /buckets/<BFS service>/<bucket>/guava-26.0-android.jar
     %jar /buckets/<BFS service>/<bucket>/jackson-core-2.9.6.jar;
     %jar /buckets/<BFS service>/<bucket>/joda-time-2.10.1.jar;
     %jar /buckets/<BFS service>/<bucket>/opencensus-api-0.18.0.jar;
@@ -55,7 +59,7 @@ Please follow the [Authenticating to a Cloud API Service article](https://cloud.
 Upload the key to BucketFS, then create a named connection:
 
 ```sql
-CREATE OR REPLACE CONNECTION BIGQUERY_CONNECTION
+CREATE OR REPLACE CONNECTION BIGQUERY_JDBC_CONNECTION
 TO 'jdbc:bigquery://https://www.googleapis.com/bigquery/v2:443;ProjectId=<your project id>;OAuthType=0;OAuthServiceAcctEmail=<service account email>;OAuthPvtKeyPath=/<path to the bucket>/<name of the key file>';
 ```    
 You can find additional information about the [JDBC connection string in the Big Query JDBC installation guide](https://www.simba.com/products/BigQuery/doc/JDBC_InstallGuide/content/jdbc/using/intro.htm);
@@ -66,15 +70,42 @@ Below you see how a Big Query Virtual Schema is created. Please note that you ha
 
 ```sql
 CREATE VIRTUAL SCHEMA <virtual schema name>
-    USING ADAPTER.JDBC_ADAPTER
+    USING SCHEMA_FOR_VS_SCRIPT.ADAPTER_SCRIPT_BIGQUERY
     WITH
     SQL_DIALECT = 'BIGQUERY'
-    CONNECTION_NAME = 'BIGQUERY_CONNECTION'
+    CONNECTION_NAME = 'BIGQUERY_JDBC_CONNECTION'
     CATALOG_NAME = '<catalog name>'
     SCHEMA_NAME = '<schema name>';
 ```
 
-## How to improve performance?
+
+## Data Types Conversion
+
+BigQuery Data Type | Supported | Converted Exasol Data Type| Known limitations
+-------------------|-----------|---------------------------|-------------------
+BOOLEAN            |  ✓        | BOOLEAN                   | 
+BYTES              |  ×        |                           | 
+DATE               |  ✓        | DATE                      | 
+DATETIME           |  ✓        | TIMESTAMP                 | 
+FLOAT              |  ✓        | DOUBLE                    | Expected range for correct mapping: -99999999.99999999 .. 99999999.99999999. 
+GEOGRAPHY          |  ✓        | VARCHAR(65535)            |
+INTEGER            |  ✓        | DECIMAL(19,0)             | 
+NUMERIC            |  ✓        | VARCHAR(2000000)          | 
+RECORD/STRUCT      |  ×        |                           | 
+STRING             |  ✓        | VARCHAR(65535)            | 
+TIME               |  ✓        | VARCHAR(16)               | 
+TIMESTAMP          |  ✓        | TIMESTAMP                 | Expected range for correct mapping: 1582-10-15 00:00:01 .. 9999-12-31 23:59:59.9999. JDBC driver maps dates before 1582-10-15 00:00:01 incorrectly.  Example of incorrect mapping: 1582-10-14 22:00:01 -> 1582-10-04 22:00:01
+
+If you need to use currently unsupported data types or find a way around known limitations, please, create a github issue in the [VS repository](https://github.com/exasol/virtual-schemas/issues).
+
+## Performance
 
 Please be aware that the current implementation of the dialect can only handle result sets with limited size (a few thousand rows).
-If you need to proceed a large amount of data, please, contact our support team and create a support ticket. We can provide another implementation of the dialect the with a speed improvement that is not available officially on our GitHub yet due to the complex installation process.
+
+## Testing information
+
+In the following matrix you find combinations of JDBC driver and dialect version that we tested.
+
+Virtual Schema Version| Big Query Version   | Driver Name                                 | Driver Version 
+----------------------|---------------------|---------------------------------------------|------------------------
+ 3.0.2                | Google BigQuery 2.0 |  Magnitude Simba JDBC driver for BigQuery   | 1.2.2.1004
diff --git a/doc/dialects/db2.md b/doc/dialects/db2.md
@@ -56,7 +56,7 @@ The SQL statement below creates the adapter script, defines the Java class that
 ```sql
 CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
   %scriptclass com.exasol.adapter.RequestDispatcher;
-  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.2.jar;
   %jar /buckets/<BFS service>/<bucket>/db2jcc4.jar;
   %jar /buckets/<BFS service>/<bucket>/db2jcc_license_cu.jar;
 /
@@ -68,7 +68,7 @@ CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
 ```sql
 CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
   %scriptclass com.exasol.adapter.RequestDispatcher;
-  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.2.jar;
   %jar /buckets/<BFS service>/<bucket>/db2jcc4.jar;
   %jar /buckets/<BFS service>/<bucket>/db2jcc_license_cu.jar;
   %jar /buckets/<BFS service>/<bucket>/db2jcc_license_cisuz.jar;

diff --git a/doc/dialects/exasol.md b/doc/dialects/exasol.md
@@ -18,7 +18,7 @@ The SQL statement below creates the adapter script, defines the Java class that
 ```sql
 CREATE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
     %scriptclass com.exasol.adapter.RequestDispatcher;
-    %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+    %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.2.jar;
 /
 ```
 

diff --git a/doc/dialects/hive.md b/doc/dialects/hive.md
@@ -47,7 +47,7 @@ The SQL statement below creates the adapter script, defines the Java class that
 ```sql
 CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
   %scriptclass com.exasol.adapter.RequestDispatcher;
-  %jar /buckets/<BFS service>/<bucket>/jars/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+  %jar /buckets/<BFS service>/<bucket>/jars/virtualschema-jdbc-adapter-dist-3.0.2.jar;
   %jar /buckets/<BFS service>/<bucket>/jars/HiveJDBC41.jar;
 /
 ```

diff --git a/doc/dialects/impala.md b/doc/dialects/impala.md
@@ -47,7 +47,7 @@ The SQL statement below creates the adapter script, defines the Java class that
 ```sql
 CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
   %scriptclass com.exasol.adapter.RequestDispatcher;
-  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.2.jar;
   %jar /buckets/<BFS service>/<bucket>/ImpalaJDBC41.jar;
 /
 ;

diff --git a/doc/dialects/mysql.md b/doc/dialects/mysql.md
@@ -51,7 +51,7 @@ The SQL statement below creates the adapter script, defines the Java class that
 ```sql
 CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
     %scriptclass com.exasol.adapter.RequestDispatcher;
-    %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+    %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.2.jar;
     %jar /buckets/<BFS service>/<bucket>/mysql-connector-java-<version>.jar;
 /
 ;

diff --git a/doc/dialects/oracle.md b/doc/dialects/oracle.md
@@ -48,7 +48,7 @@ The SQL statement below creates the adapter script, defines the Java class that
 ```sql
 CREATE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
   %scriptclass com.exasol.adapter.RequestDispatcher;
-  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.2.jar;
   %jar /buckets/<BFS service>/<bucket>/ojdbc<JDBC driver version>.jar;
 /
 ;

diff --git a/doc/dialects/postgresql.md b/doc/dialects/postgresql.md
@@ -24,7 +24,7 @@ The SQL statement below creates the adapter script, defines the Java class that
 ```sql
 CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
   %scriptclass com.exasol.adapter.RequestDispatcher;
-  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.2.jar;
   %jar /buckets/<BFS service>/<bucket>/postgresql-<version>.jar;
 /
 ```

diff --git a/doc/dialects/redshift.md b/doc/dialects/redshift.md
@@ -51,7 +51,7 @@ The SQL statement below creates the adapter script, defines the Java class that
 ```sql
 CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
     %scriptclass com.exasol.adapter.RequestDispatcher;
-    %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+    %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.2.jar;
     %jar /buckets/<BFS service>/<bucket>/RedshiftJDBC42-<JDBC driver version>.jar;
 /
 ;

diff --git a/doc/dialects/saphana.md b/doc/dialects/saphana.md
@@ -47,7 +47,7 @@ The SQL statement below creates the adapter script, defines the Java class that
 ```sql
 CREATE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
      %scriptclass com.exasol.adapter.RequestDispatcher;
-     %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+     %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.2.jar;
      %jar /buckets/<BFS service>/<bucket>/ngdbc-<JDBC driver version>.jar;
 /
 ;

diff --git a/doc/dialects/sql_server.md b/doc/dialects/sql_server.md
@@ -24,7 +24,7 @@ The SQL statement below creates the adapter script, defines the Java class that
 ```sql
 CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
   %scriptclass com.exasol.adapter.RequestDispatcher;
-  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.2.jar;
   %jar /buckets/<BFS service>/<bucket>/jtds.jar;
 /
 ```

diff --git a/doc/dialects/sybase.md b/doc/dialects/sybase.md
@@ -29,7 +29,7 @@ The SQL statement below creates the adapter script, defines the Java class that
 ```sql
 CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
   %scriptclass com.exasol.adapter.RequestDispatcher;
-  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.2.jar;
   %jar /buckets/<BFS service>/<bucket>/jtds-<version>.jar;
 /
 ```

diff --git a/doc/dialects/teradata.md b/doc/dialects/teradata.md
@@ -47,7 +47,7 @@ The SQL statement below creates the adapter script, defines the Java class that
 ```sql
 CREATE OR REPLACE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
   %scriptclass com.exasol.adapter.RequestDispatcher;
-  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+  %jar /buckets/<BFS service>/<bucket>/virtualschema-jdbc-adapter-dist-3.0.2.jar;
   %jar /buckets/<BFS service>/<bucket>/terajdbc4.jar;
   %jar /buckets/<BFS service>/<bucket>/tdgssconfig.jar;
 /

diff --git a/doc/user-guide/deploying_the_virtual_schema_adapter.md b/doc/user-guide/deploying_the_virtual_schema_adapter.md
@@ -23,7 +23,7 @@ cd virtual-schemas/jdbc-adapter/
 mvn clean -DskipTests package
 ```
 
-The resulting fat JAR is stored in `virtualschema-jdbc-adapter-dist/target/virtualschema-jdbc-adapter-dist-3.0.1.jar`.
+The resulting fat JAR is stored in `virtualschema-jdbc-adapter-dist/target/virtualschema-jdbc-adapter-dist-3.0.2.jar`.
 
 ## Uploading the Adapter JAR Archive
 
@@ -46,8 +46,8 @@ Following steps are required to upload a file to a bucket:
 1. Now upload the file into this bucket, e.g. using curl (adapt the hostname, BucketFS port, bucket name and bucket write password).
 
 ```bash
-curl -X PUT -T virtualschema-jdbc-adapter-dist/target/virtualschema-jdbc-adapter-dist-3.0.1.jar \
- http://w:[email protected]:2580/bucket1/virtualschema-jdbc-adapter-dist-3.0.1.jar
+curl -X PUT -T virtualschema-jdbc-adapter-dist/target/virtualschema-jdbc-adapter-dist-3.0.2.jar \
+ http://w:[email protected]:2580/bucket1/virtualschema-jdbc-adapter-dist-3.0.2.jar
 ```
 
 If you later need to change the bucket passwords, select the bucket and click "Edit".
@@ -84,7 +84,7 @@ CREATE SCHEMA ADAPTER;
 
 CREATE JAVA ADAPTER SCRIPT ADAPTER.JDBC_ADAPTER AS
   %scriptclass com.exasol.adapter.RequestDispatcher;
-  %jar /buckets/your-bucket-fs/your-bucket/virtualschema-jdbc-adapter-dist-3.0.1.jar;
+  %jar /buckets/your-bucket-fs/your-bucket/virtualschema-jdbc-adapter-dist-3.0.2.jar;
   %jar /buckets/your-bucket-fs/your-bucket/<JDBC driver>.jar;
 /
 ```

diff --git a/pom.xml b/pom.xml
@@ -7,7 +7,7 @@
     <version>${product.version}</version>
     <name>Virtual Schema JDBC Adapter</name>
     <properties>
-        <product.version>3.0.1</product.version>
+        <product.version>3.0.2</product.version>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <java.version>11</java.version>

diff --git a/src/main/java/com/exasol/adapter/dialects/bigquery/BigQueryColumnMetadataReader.java b/src/main/java/com/exasol/adapter/dialects/bigquery/BigQueryColumnMetadataReader.java
@@ -0,0 +1,35 @@
+package com.exasol.adapter.dialects.bigquery;
+
+import java.sql.Connection;
+import java.sql.Types;
+
+import com.exasol.adapter.AdapterProperties;
+import com.exasol.adapter.dialects.IdentifierConverter;
+import com.exasol.adapter.jdbc.BaseColumnMetadataReader;
+import com.exasol.adapter.jdbc.JdbcTypeDescription;
+import com.exasol.adapter.metadata.DataType;
+
+/**
+ * This class implements BigQuery-specific reading of column metadata.
+ */
+public class BigQueryColumnMetadataReader extends BaseColumnMetadataReader {
+    /**
+     * Create a new instance of the {@link BigQueryColumnMetadataReader}.
+     *
+     * @param connection          connection to the remote data source
+     * @param properties          user-defined adapter properties
+     * @param identifierConverter converter between source and Exasol identifiers
+     */
+    public BigQueryColumnMetadataReader(final Connection connection, final AdapterProperties properties,
+            final IdentifierConverter identifierConverter) {
+        super(connection, properties, identifierConverter);
+    }
+
+    @Override
+    public DataType mapJdbcType(final JdbcTypeDescription jdbcTypeDescription) {
+        if (jdbcTypeDescription.getJdbcType() == Types.TIME) {
+            return DataType.createVarChar(16, DataType.ExaCharset.UTF8);
+        }
+        return super.mapJdbcType(jdbcTypeDescription);
+    }
+}
diff --git a/src/main/java/com/exasol/adapter/dialects/bigquery/BigQueryMetadataReader.java b/src/main/java/com/exasol/adapter/dialects/bigquery/BigQueryMetadataReader.java
@@ -25,7 +25,7 @@ public BigQueryMetadataReader(final Connection connection, final AdapterProperti
 
     @Override
     protected ColumnMetadataReader createColumnMetadataReader() {
-        return new BaseColumnMetadataReader(this.connection, this.properties, this.identifierConverter);
+        return new BigQueryColumnMetadataReader(this.connection, this.properties, this.identifierConverter);
     }
 
     @Override