diff --git a/tests/resources/bigquery_test_tables.sql b/tests/resources/bigquery_test_tables.sql index 580525b9..3468a7b3 100644 --- a/tests/resources/bigquery_test_tables.sql +++ b/tests/resources/bigquery_test_tables.sql @@ -1,3 +1,16 @@ +-- Copyright 2021 Google LLC +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. -- BigQuery table with several different numeric datatypes with the same value CREATE OR REPLACE TABLE pso_data_validator.test_data_types AS @@ -6,7 +19,7 @@ SELECT CAST(2 AS INT64) int_type, CAST(2 AS DECIMAL) decimal_type, CAST(2 AS STRING) text_type, - CAST('2021-01-01 00:00:00' AS TIMESTAMP) timestamp_type + CAST('2021-01-01 00:00:00' AS TIMESTAMP) timestamp_type; -- Core data types test table, to be kept in sync with same table in other SQL engines @@ -52,19 +65,17 @@ INSERT INTO `pso_data_validator`.`dvt_core_types` VALUES ,DATE '1970-01-03',DATETIME '1970-01-03 00:00:03' ,TIMESTAMP '1970-01-03 00:00:03-03:00'); -CREATE VIEW `pso_data_validator`.`dvt_core_types_vw` AS +CREATE OR REPLACE VIEW `pso_data_validator`.`dvt_core_types_vw` AS SELECT * FROM `pso_data_validator`.`dvt_core_types`; -DROP TABLE `pso_data_validator`.`dvt_null_not_null`; -CREATE TABLE `pso_data_validator`.`dvt_null_not_null` +CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_null_not_null` ( col_nn DATETIME NOT NULL , col_nullable DATETIME , col_src_nn_trg_n DATETIME , col_src_n_trg_nn DATETIME NOT NULL ) OPTIONS (description='Nullable integration test table, BigQuery is assumed to be a DVT target (not source)'); -DROP TABLE `pso_data_validator`.`dvt_large_decimals`; -CREATE TABLE `pso_data_validator`.`dvt_large_decimals` +CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_large_decimals` ( id BIGNUMERIC(38) NOT NULL , col_data STRING(10) , col_dec_18 INT64 @@ -108,8 +119,7 @@ INSERT INTO `pso_data_validator`.`dvt_large_decimals` VALUES ,BIGNUMERIC '12345678.123456789012345678901234567890' ,987654321012345670,12345678901234567.0); -DROP TABLE `pso_data_validator`.`dvt_binary`; -CREATE TABLE `pso_data_validator`.`dvt_binary` +CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_binary` ( binary_id BYTES(16) NOT NULL , int_id INT64 NOT NULL , other_data STRING(100) @@ -121,8 +131,7 @@ INSERT INTO `pso_data_validator`.`dvt_binary` VALUES (CAST('DVT-key-4' AS BYTES), 4, 'Row 4'), (CAST('DVT-key-5' AS BYTES), 5, 'Row 5'); -DROP TABLE `pso_data_validator`.`dvt_string_id`; -CREATE TABLE `pso_data_validator`.`dvt_string_id` +CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_string_id` ( id STRING(15) NOT NULL , other_data STRING(100) ) OPTIONS (description='Integration test table used to test string pk matching.'); @@ -133,9 +142,8 @@ INSERT INTO `pso_data_validator`.`dvt_string_id` VALUES ('DVT-key-4', 'Row 4'), ('DVT-key-5', 'Row 5'); -DROP TABLE `pso_data_validator`.`dvt_char_id`; -- BigQuery does not have a specific padded CHAR data type. -CREATE TABLE `pso_data_validator`.`dvt_char_id` +CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_char_id` ( id STRING(6) NOT NULL , other_data STRING(100) ) OPTIONS (description='Integration test table used to test CHAR pk matching.'); @@ -146,8 +154,7 @@ INSERT INTO `pso_data_validator`.`dvt_char_id` VALUES ('DVT4 ', 'Row 4 '), ('DVT5 ', 'Row 5'); -DROP TABLE `pso_data_validator`.`dvt_time_table`; -CREATE TABLE `pso_data_validator`.`dvt_time_table` +CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_time_table` ( id INTEGER NOT NULL , col_time TIME ) OPTIONS (description='Integration test table used to test Time data type'); @@ -156,8 +163,7 @@ INSERT INTO `pso_data_validator`.`dvt_time_table` VALUES (2, '04:02:00'), (3, '08:01:07'); -DROP TABLE `pso_data_validator`.`dvt_latin`; -CREATE TABLE `pso_data_validator`.`dvt_latin` +CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_latin` ( id INT64 , words STRING ) OPTIONS (description='Integration test table used to test latin characters.'); @@ -169,8 +175,7 @@ INSERT INTO `pso_data_validator`.`dvt_latin` (id, words) VALUES (4, 'SAINT-RENÉ'), (5, 'SAINTE-ANE-DE-LA-PÉ'); -DROP TABLE `pso_data_validator`.`dvt_pangrams`; -CREATE TABLE `pso_data_validator`.`dvt_pangrams` +CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_pangrams` ( id INT64 , lang STRING(100) , words STRING(1000) @@ -189,8 +194,7 @@ INSERT INTO `pso_data_validator`.`dvt_pangrams` VALUES (5,'Turkish', 'Pijamalı hasta yağız şoföre çabucak güvendi', 'The sick person in pyjamas quickly trusted the swarthy driver'); -DROP TABLE `pso_data_validator`.`dvt_many_cols`; -CREATE TABLE `pso_data_validator`.`dvt_many_cols` +CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_many_cols` ( id INT64 , col_001 STRING , col_002 STRING @@ -594,9 +598,66 @@ CREATE TABLE `pso_data_validator`.`dvt_many_cols` ) OPTIONS (description='Integration test table used to test validating many columns.'); INSERT INTO `pso_data_validator`.`dvt_many_cols` (id) VALUES (1); +CREATE OR REPLACE TABLE `pso_data_validator`.`test_generate_partitions` ( + course_id STRING +, quarter_id INT64 +, student_id INT64 +, grade FLOAT64 +, registration_timestamp TIMESTAMP +, registration_date DATE +) OPTIONS (description='Table for testing generate-table-partitions feature, consists of rows with a composite primary key'); +INSERT INTO `pso_data_validator`.`test_generate_partitions` VALUES +('ALG001', 1, 5678, 3.5, NULL, NULL), +('TRI001', 1, 5678, 3.5, NULL, NULL), +('GEO001', 1, 5678, 3.5, NULL, NULL), +('TRI001', 1, 9012, 2.3, NULL, NULL), +('ALG001', 1, 9012, 2.3, NULL, NULL), +('GEO001', 1, 9012, 2.3, NULL, NULL), +('ALG001', 1, 1234, 2.1, NULL, NULL), +('TRI001', 1, 1234, 2.1, NULL, NULL), +('GEO001', 1, 1234, 2.1, NULL, NULL), +('TRI001', 2, 1234, 3.5, NULL, NULL), +('GEO001', 2, 9012, 3.5, NULL, NULL), +('GEO001', 2, 1234, 3.5, NULL, NULL), +('ALG001', 2, 9012, 3.5, NULL, NULL), +('ALG001', 2, 1234, 3.5, NULL, NULL), +('TRI001', 2, 9012, 3.5, NULL, NULL), +('TRI001', 2, 5678, 2.6, NULL, NULL), +('ALG001', 2, 5678, 2.6, NULL, NULL), +('GEO001', 2, 5678, 2.6, NULL, NULL), +('GEO001', 3, 5678, 3.5, NULL, NULL), +('ALG001', 3, 5678, 3.5, NULL, NULL), +('TRI001', 3, 5678, 3.5, NULL, NULL), +('ALG001', 3, 9012, 2.8, NULL, NULL), +('TRI001', 3, 9012, 2.8, NULL, NULL), +('GEO001', 3, 9012, 2.8, NULL, NULL), +('TRI001', 3, 1234, 2.7, NULL, NULL), +('ALG001', 3, 1234, 2.7, NULL, NULL), +('GEO001', 3, 1234, 2.7, NULL, NULL); + +CREATE OR REPLACE TABLE `pso_data_validator`.`test_generate_partitions_for_date_timestamp` ( + col_timestamp TIMESTAMP +, col_date DATE +); +INSERT INTO `pso_data_validator`.`test_generate_partitions_for_date_timestamp` VALUES +('2023-07-03 12:45:00.000000 UTC', '2023-07-03'), +('2023-07-06 16:00:00.000000 UTC', '2023-07-06'), +('2023-07-09 11:00:00.000000 UTC', '2023-07-09'), +('2023-07-01 10:00:00.000000 UTC', '2023-07-01'), +('2023-07-07 08:45:00.000000 UTC', '2023-07-07'), +('2023-07-08 13:20:00.000000 UTC', '2023-07-08'), +('2023-07-12 12:30:00.000000 UTC', '2023-07-12'), +('2023-07-13 09:45:00.000000 UTC', '2023-07-13'), +('2023-07-02 11:30:00.000000 UTC', '2023-07-02'), +('2023-07-04 09:15:00.000000 UTC', '2023-07-04'), +('2023-07-10 15:30:00.000000 UTC', '2023-07-10'), +('2023-07-15 16:45:00.000000 UTC', '2023-07-15'), +('2023-07-05 14:30:00.000000 UTC', '2023-07-05'), +('2023-07-11 10:15:00.000000 UTC', '2023-07-11'), +('2023-07-14 14:00:00.000000 UTC', '2023-07-14'); + -- Name should contain $ and # but this is not supported in BigQuery: `pso_data_validator`.`dvt-identifier$_#`; -DROP TABLE `pso_data_validator`.`dvt-identifier___`; -CREATE TABLE `pso_data_validator`.`dvt-identifier___` +CREATE OR REPLACE TABLE `pso_data_validator`.`dvt-identifier___` ( id INT64 NOT NULL , `col#hash` STRING , `col_dollar` STRING -- Name should contain $ but this is not supported in BigQuery @@ -610,8 +671,7 @@ INSERT INTO `pso_data_validator`.`dvt-identifier___` VALUES (3,'#','$','-','@',' INSERT INTO `pso_data_validator`.`dvt-identifier___` VALUES (4,'#','$','-','@','Row 4'); INSERT INTO `pso_data_validator`.`dvt-identifier___` VALUES (5,'#','$','-','@','Row 5'); -DROP TABLE `pso_data_validator`.`dvt_bool`; -CREATE TABLE `pso_data_validator`.`dvt_bool` +CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_bool` ( id INT64 NOT NULL , col_bool_dec BOOLEAN , col_bool_int BOOLEAN @@ -622,8 +682,7 @@ INSERT INTO `pso_data_validator`.`dvt_bool` VALUES (1,true,true,true,true); INSERT INTO `pso_data_validator`.`dvt_bool` VALUES (2,false,false,false,false); -- BigQuery stores UUIDs in STRING data type according to its utility function called GENERATE_UUID(). -DROP TABLE `pso_data_validator`.`dvt_uuid_id`; -CREATE TABLE `pso_data_validator`.`dvt_uuid_id` +CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_uuid_id` ( id STRING NOT NULL , col_uuid STRING , col_data STRING @@ -632,8 +691,7 @@ INSERT INTO `pso_data_validator`.`dvt_uuid_id` VALUES ('387bdc3b-2184-43b2-8ec2-3ac791c5b0f1','387bdc3b-2184-43b2-8ec2-3ac791c5b0f1','A'), ('397bdc3b-2184-43b2-8ec2-3ac791c5b0f1','397bdc3b-2184-43b2-8ec2-3ac791c5b0f1','B'); -DROP TABLE `pso_data_validator`.`dvt_group_by_timestamp`; -CREATE TABLE `pso_data_validator`.`dvt_group_by_timestamp` +CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_group_by_timestamp` ( id INT64 NOT NULL , group_id INT64 , col_date DATE @@ -647,8 +705,7 @@ INSERT INTO `pso_data_validator`.`dvt_group_by_timestamp` VALUES (5,2,DATE'2022-02-02',DATETIME'2022-02-02 13:00:00'), (6,3,DATE'2023-03-03',DATETIME'2023-03-03 12:00:00'); -DROP TABLE `pso_data_validator`.`dvt_tricky_dates`; -CREATE TABLE `pso_data_validator`.`dvt_tricky_dates` ( +CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_tricky_dates` ( id INT64 NOT NULL , col_dt_low DATE , col_dt_epoch DATE diff --git a/tests/resources/mysql_test_tables.sql b/tests/resources/mysql_test_tables.sql index ce4b81eb..1ce7c1f4 100644 --- a/tests/resources/mysql_test_tables.sql +++ b/tests/resources/mysql_test_tables.sql @@ -69,7 +69,7 @@ CREATE TABLE `pso_data_validator`.`dvt_core_types` , col_float64 double , col_varchar_30 varchar(30) , col_char_2 char(2) -, col_string varchar(21000) +, col_string text , col_date date , col_datetime datetime(3) , col_tstz timestamp(3) diff --git a/tests/resources/postgresql_test_tables.sql b/tests/resources/postgresql_test_tables.sql index be2fec37..68df55d6 100644 --- a/tests/resources/postgresql_test_tables.sql +++ b/tests/resources/postgresql_test_tables.sql @@ -13,7 +13,7 @@ -- limitations under the License. CREATE SCHEMA pso_data_validator; -DROP TABLE IF EXISTS pso_data_validator.dvt_core_types; +DROP TABLE IF EXISTS pso_data_validator.dvt_core_types CASCADE; CREATE TABLE pso_data_validator.dvt_core_types ( id int NOT NULL PRIMARY KEY , col_int8 smallint @@ -51,7 +51,7 @@ INSERT INTO pso_data_validator.dvt_core_types VALUES ,DATE'1970-01-03',TIMESTAMP'1970-01-03 00:00:03' ,TIMESTAMP WITH TIME ZONE'1970-01-03 00:00:03 -03:00'); -CREATE VIEW pso_data_validator.dvt_core_types_vw AS +CREATE OR REPLACE VIEW pso_data_validator.dvt_core_types_vw AS SELECT * FROM pso_data_validator.dvt_core_types; DROP TABLE IF EXISTS pso_data_validator.dvt_ora2pg_types; @@ -215,7 +215,7 @@ CREATE TABLE pso_data_validator.dvt_pg_types ); COMMENT ON TABLE pso_data_validator.dvt_pg_types IS 'PostgreSQL data types integration test table'; -CREATE EXTENSION pgcrypto; +CREATE EXTENSION IF NOT EXISTS pgcrypto; INSERT INTO pso_data_validator.dvt_pg_types (col_int2,col_int4,col_int8,col_dec,col_dec_10_2 --,col_money @@ -311,7 +311,7 @@ INSERT INTO pso_data_validator.dvt_binary VALUES (CAST('DVT-key-4' AS bytea), 4, 'Row 4'), (CAST('DVT-key-5' AS bytea), 5, 'Row 5'); -DROP TABLE pso_data_validator.dvt_char_id; +DROP TABLE IF EXISTS pso_data_validator.dvt_char_id; CREATE TABLE pso_data_validator.dvt_char_id ( id char(6) NOT NULL PRIMARY KEY , other_data varchar(100) @@ -324,7 +324,7 @@ INSERT INTO pso_data_validator.dvt_char_id VALUES ('DVT4', 'Row 4'), ('DVT5', 'Row 5'); -DROP TABLE pso_data_validator.dvt_pangrams; +DROP TABLE IF EXISTS pso_data_validator.dvt_pangrams; CREATE TABLE pso_data_validator.dvt_pangrams ( id int , lang varchar(100) @@ -346,7 +346,7 @@ INSERT INTO pso_data_validator.dvt_pangrams VALUES (5,'Turkish', 'Pijamalı hasta yağız şoföre çabucak güvendi', 'The sick person in pyjamas quickly trusted the swarthy driver'); -DROP TABLE pso_data_validator.dvt_many_cols; +DROP TABLE IF EXISTS pso_data_validator.dvt_many_cols; CREATE TABLE pso_data_validator.dvt_many_cols ( id decimal(5) , col_001 varchar(2) @@ -752,7 +752,7 @@ CREATE TABLE pso_data_validator.dvt_many_cols COMMENT ON TABLE pso_data_validator.dvt_many_cols IS 'Integration test table used to test validating many columns.'; INSERT INTO pso_data_validator.dvt_many_cols (id) values (1); -DROP TABLE pso_data_validator."dvt-identifier$_#"; +DROP TABLE IF EXISTS pso_data_validator."dvt-identifier$_#"; CREATE TABLE pso_data_validator."dvt-identifier$_#" ( id int NOT NULL PRIMARY KEY , "col#hash" varchar(10) @@ -768,7 +768,7 @@ INSERT INTO pso_data_validator."dvt-identifier$_#" VALUES (3,'#','$','-','@','Ro INSERT INTO pso_data_validator."dvt-identifier$_#" VALUES (4,'#','$','-','@','Row 4'); INSERT INTO pso_data_validator."dvt-identifier$_#" VALUES (5,'#','$','-','@','Row 5'); -DROP TABLE pso_data_validator.dvt_bool; +DROP TABLE IF EXISTS pso_data_validator.dvt_bool; CREATE TABLE pso_data_validator.dvt_bool ( id int NOT NULL PRIMARY KEY , col_bool_dec boolean @@ -779,7 +779,7 @@ COMMENT ON TABLE pso_data_validator.dvt_bool IS 'Integration test table used to INSERT INTO pso_data_validator.dvt_bool VALUES (1,true,true,true,true); INSERT INTO pso_data_validator.dvt_bool VALUES (2,false,false,false,false); -DROP TABLE pso_data_validator.dvt_uuid_id; +DROP TABLE IF EXISTS pso_data_validator.dvt_uuid_id; CREATE TABLE pso_data_validator.dvt_uuid_id ( id uuid NOT NULL PRIMARY KEY , col_uuid uuid @@ -789,7 +789,7 @@ INSERT INTO pso_data_validator.dvt_uuid_id VALUES (uuid('387bdc3b218443b28ec23ac791c5b0f1'),uuid('387bdc3b218443b28ec23ac791c5b0f1'),'A'), (uuid('397bdc3b218443b28ec23ac791c5b0f1'),uuid('397bdc3b218443b28ec23ac791c5b0f1'),'B'); -DROP TABLE pso_data_validator.dvt_group_by_timestamp; +DROP TABLE IF EXISTS pso_data_validator.dvt_group_by_timestamp; CREATE TABLE pso_data_validator.dvt_group_by_timestamp ( id int NOT NULL PRIMARY KEY , group_id int