Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

tests: Update DDL scripts for test tables #1416

Draft
wants to merge 3 commits into
base: develop
Choose a base branch
from
Draft

tests: Update DDL scripts for test tables #1416

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f714ce9
Updated BQ, MySQL, Postgres files
helensilva14 Jan 24, 2025
185bba2
Add BQ test tables for generate-table-partitions
helensilva14 Jan 27, 2025
159a3b0
Merge branch 'develop' into update-sql-scripts-for-test-tables
helensilva14 Jan 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 87 additions & 30 deletions tests/resources/bigquery_test_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,16 @@
-- Copyright 2021 Google LLC
--
-- Licensed under the Apache License, Version 2.0 (the "License");
-- you may not use this file except in compliance with the License.
-- You may obtain a copy of the License at
--
-- http://www.apache.org/licenses/LICENSE-2.0
--
-- Unless required by applicable law or agreed to in writing, software
-- distributed under the License is distributed on an "AS IS" BASIS,
-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-- See the License for the specific language governing permissions and
-- limitations under the License.

-- BigQuery table with several different numeric datatypes with the same value
CREATE OR REPLACE TABLE pso_data_validator.test_data_types AS
Expand All @@ -6,7 +19,7 @@ SELECT
CAST(2 AS INT64) int_type,
CAST(2 AS DECIMAL) decimal_type,
CAST(2 AS STRING) text_type,
CAST('2021-01-01 00:00:00' AS TIMESTAMP) timestamp_type
CAST('2021-01-01 00:00:00' AS TIMESTAMP) timestamp_type;


-- Core data types test table, to be kept in sync with same table in other SQL engines
Expand Down Expand Up @@ -52,19 +65,17 @@ INSERT INTO `pso_data_validator`.`dvt_core_types` VALUES
,DATE '1970-01-03',DATETIME '1970-01-03 00:00:03'
,TIMESTAMP '1970-01-03 00:00:03-03:00');

CREATE VIEW `pso_data_validator`.`dvt_core_types_vw` AS
CREATE OR REPLACE VIEW `pso_data_validator`.`dvt_core_types_vw` AS
SELECT * FROM `pso_data_validator`.`dvt_core_types`;

DROP TABLE `pso_data_validator`.`dvt_null_not_null`;
CREATE TABLE `pso_data_validator`.`dvt_null_not_null`
CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_null_not_null`
( col_nn DATETIME NOT NULL
, col_nullable DATETIME
, col_src_nn_trg_n DATETIME
, col_src_n_trg_nn DATETIME NOT NULL
) OPTIONS (description='Nullable integration test table, BigQuery is assumed to be a DVT target (not source)');

DROP TABLE `pso_data_validator`.`dvt_large_decimals`;
CREATE TABLE `pso_data_validator`.`dvt_large_decimals`
CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_large_decimals`
( id BIGNUMERIC(38) NOT NULL
, col_data STRING(10)
, col_dec_18 INT64
Expand Down Expand Up @@ -108,8 +119,7 @@ INSERT INTO `pso_data_validator`.`dvt_large_decimals` VALUES
,BIGNUMERIC '12345678.123456789012345678901234567890'
,987654321012345670,12345678901234567.0);

DROP TABLE `pso_data_validator`.`dvt_binary`;
CREATE TABLE `pso_data_validator`.`dvt_binary`
CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_binary`
( binary_id BYTES(16) NOT NULL
, int_id INT64 NOT NULL
, other_data STRING(100)
Expand All @@ -121,8 +131,7 @@ INSERT INTO `pso_data_validator`.`dvt_binary` VALUES
(CAST('DVT-key-4' AS BYTES), 4, 'Row 4'),
(CAST('DVT-key-5' AS BYTES), 5, 'Row 5');

DROP TABLE `pso_data_validator`.`dvt_string_id`;
CREATE TABLE `pso_data_validator`.`dvt_string_id`
CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_string_id`
( id STRING(15) NOT NULL
, other_data STRING(100)
) OPTIONS (description='Integration test table used to test string pk matching.');
Expand All @@ -133,9 +142,8 @@ INSERT INTO `pso_data_validator`.`dvt_string_id` VALUES
('DVT-key-4', 'Row 4'),
('DVT-key-5', 'Row 5');

DROP TABLE `pso_data_validator`.`dvt_char_id`;
-- BigQuery does not have a specific padded CHAR data type.
CREATE TABLE `pso_data_validator`.`dvt_char_id`
CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_char_id`
( id STRING(6) NOT NULL
, other_data STRING(100)
) OPTIONS (description='Integration test table used to test CHAR pk matching.');
Expand All @@ -146,8 +154,7 @@ INSERT INTO `pso_data_validator`.`dvt_char_id` VALUES
('DVT4 ', 'Row 4 '),
('DVT5 ', 'Row 5');

DROP TABLE `pso_data_validator`.`dvt_time_table`;
CREATE TABLE `pso_data_validator`.`dvt_time_table`
CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_time_table`
( id INTEGER NOT NULL
, col_time TIME
) OPTIONS (description='Integration test table used to test Time data type');
Expand All @@ -156,8 +163,7 @@ INSERT INTO `pso_data_validator`.`dvt_time_table` VALUES
(2, '04:02:00'),
(3, '08:01:07');

DROP TABLE `pso_data_validator`.`dvt_latin`;
CREATE TABLE `pso_data_validator`.`dvt_latin`
CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_latin`
( id INT64
, words STRING
) OPTIONS (description='Integration test table used to test latin characters.');
Expand All @@ -169,8 +175,7 @@ INSERT INTO `pso_data_validator`.`dvt_latin` (id, words) VALUES
(4, 'SAINT-RENÉ'),
(5, 'SAINTE-ANE-DE-LA-PÉ');

DROP TABLE `pso_data_validator`.`dvt_pangrams`;
CREATE TABLE `pso_data_validator`.`dvt_pangrams`
CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_pangrams`
( id INT64
, lang STRING(100)
, words STRING(1000)
Expand All @@ -189,8 +194,7 @@ INSERT INTO `pso_data_validator`.`dvt_pangrams` VALUES
(5,'Turkish', 'Pijamalı hasta yağız şoföre çabucak güvendi',
'The sick person in pyjamas quickly trusted the swarthy driver');

DROP TABLE `pso_data_validator`.`dvt_many_cols`;
CREATE TABLE `pso_data_validator`.`dvt_many_cols`
CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_many_cols`
( id INT64
, col_001 STRING
, col_002 STRING
Expand Down Expand Up @@ -594,9 +598,66 @@ CREATE TABLE `pso_data_validator`.`dvt_many_cols`
) OPTIONS (description='Integration test table used to test validating many columns.');
INSERT INTO `pso_data_validator`.`dvt_many_cols` (id) VALUES (1);

CREATE OR REPLACE TABLE `pso_data_validator`.`test_generate_partitions` (
course_id STRING
, quarter_id INT64
, student_id INT64
, grade FLOAT64
, registration_timestamp TIMESTAMP
, registration_date DATE
) OPTIONS (description='Table for testing generate-table-partitions feature, consists of rows with a composite primary key');
INSERT INTO `pso_data_validator`.`test_generate_partitions` VALUES
('ALG001', 1, 5678, 3.5, NULL, NULL),
('TRI001', 1, 5678, 3.5, NULL, NULL),
('GEO001', 1, 5678, 3.5, NULL, NULL),
('TRI001', 1, 9012, 2.3, NULL, NULL),
('ALG001', 1, 9012, 2.3, NULL, NULL),
('GEO001', 1, 9012, 2.3, NULL, NULL),
('ALG001', 1, 1234, 2.1, NULL, NULL),
('TRI001', 1, 1234, 2.1, NULL, NULL),
('GEO001', 1, 1234, 2.1, NULL, NULL),
('TRI001', 2, 1234, 3.5, NULL, NULL),
('GEO001', 2, 9012, 3.5, NULL, NULL),
('GEO001', 2, 1234, 3.5, NULL, NULL),
('ALG001', 2, 9012, 3.5, NULL, NULL),
('ALG001', 2, 1234, 3.5, NULL, NULL),
('TRI001', 2, 9012, 3.5, NULL, NULL),
('TRI001', 2, 5678, 2.6, NULL, NULL),
('ALG001', 2, 5678, 2.6, NULL, NULL),
('GEO001', 2, 5678, 2.6, NULL, NULL),
('GEO001', 3, 5678, 3.5, NULL, NULL),
('ALG001', 3, 5678, 3.5, NULL, NULL),
('TRI001', 3, 5678, 3.5, NULL, NULL),
('ALG001', 3, 9012, 2.8, NULL, NULL),
('TRI001', 3, 9012, 2.8, NULL, NULL),
('GEO001', 3, 9012, 2.8, NULL, NULL),
('TRI001', 3, 1234, 2.7, NULL, NULL),
('ALG001', 3, 1234, 2.7, NULL, NULL),
('GEO001', 3, 1234, 2.7, NULL, NULL);

CREATE OR REPLACE TABLE `pso_data_validator`.`test_generate_partitions_for_date_timestamp` (
col_timestamp TIMESTAMP
, col_date DATE
);
INSERT INTO `pso_data_validator`.`test_generate_partitions_for_date_timestamp` VALUES
('2023-07-03 12:45:00.000000 UTC', '2023-07-03'),
('2023-07-06 16:00:00.000000 UTC', '2023-07-06'),
('2023-07-09 11:00:00.000000 UTC', '2023-07-09'),
('2023-07-01 10:00:00.000000 UTC', '2023-07-01'),
('2023-07-07 08:45:00.000000 UTC', '2023-07-07'),
('2023-07-08 13:20:00.000000 UTC', '2023-07-08'),
('2023-07-12 12:30:00.000000 UTC', '2023-07-12'),
('2023-07-13 09:45:00.000000 UTC', '2023-07-13'),
('2023-07-02 11:30:00.000000 UTC', '2023-07-02'),
('2023-07-04 09:15:00.000000 UTC', '2023-07-04'),
('2023-07-10 15:30:00.000000 UTC', '2023-07-10'),
('2023-07-15 16:45:00.000000 UTC', '2023-07-15'),
('2023-07-05 14:30:00.000000 UTC', '2023-07-05'),
('2023-07-11 10:15:00.000000 UTC', '2023-07-11'),
('2023-07-14 14:00:00.000000 UTC', '2023-07-14');

-- Name should contain $ and # but this is not supported in BigQuery: `pso_data_validator`.`dvt-identifier$_#`;
DROP TABLE `pso_data_validator`.`dvt-identifier___`;
CREATE TABLE `pso_data_validator`.`dvt-identifier___`
CREATE OR REPLACE TABLE `pso_data_validator`.`dvt-identifier___`
( id INT64 NOT NULL
, `col#hash` STRING
, `col_dollar` STRING -- Name should contain $ but this is not supported in BigQuery
Expand All @@ -610,8 +671,7 @@ INSERT INTO `pso_data_validator`.`dvt-identifier___` VALUES (3,'#','$','-','@','
INSERT INTO `pso_data_validator`.`dvt-identifier___` VALUES (4,'#','$','-','@','Row 4');
INSERT INTO `pso_data_validator`.`dvt-identifier___` VALUES (5,'#','$','-','@','Row 5');

DROP TABLE `pso_data_validator`.`dvt_bool`;
CREATE TABLE `pso_data_validator`.`dvt_bool`
CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_bool`
( id INT64 NOT NULL
, col_bool_dec BOOLEAN
, col_bool_int BOOLEAN
Expand All @@ -622,8 +682,7 @@ INSERT INTO `pso_data_validator`.`dvt_bool` VALUES (1,true,true,true,true);
INSERT INTO `pso_data_validator`.`dvt_bool` VALUES (2,false,false,false,false);

-- BigQuery stores UUIDs in STRING data type according to its utility function called GENERATE_UUID().
DROP TABLE `pso_data_validator`.`dvt_uuid_id`;
CREATE TABLE `pso_data_validator`.`dvt_uuid_id`
CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_uuid_id`
( id STRING NOT NULL
, col_uuid STRING
, col_data STRING
Expand All @@ -632,8 +691,7 @@ INSERT INTO `pso_data_validator`.`dvt_uuid_id` VALUES
('387bdc3b-2184-43b2-8ec2-3ac791c5b0f1','387bdc3b-2184-43b2-8ec2-3ac791c5b0f1','A'),
('397bdc3b-2184-43b2-8ec2-3ac791c5b0f1','397bdc3b-2184-43b2-8ec2-3ac791c5b0f1','B');

DROP TABLE `pso_data_validator`.`dvt_group_by_timestamp`;
CREATE TABLE `pso_data_validator`.`dvt_group_by_timestamp`
CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_group_by_timestamp`
( id INT64 NOT NULL
, group_id INT64
, col_date DATE
Expand All @@ -647,8 +705,7 @@ INSERT INTO `pso_data_validator`.`dvt_group_by_timestamp` VALUES
(5,2,DATE'2022-02-02',DATETIME'2022-02-02 13:00:00'),
(6,3,DATE'2023-03-03',DATETIME'2023-03-03 12:00:00');

DROP TABLE `pso_data_validator`.`dvt_tricky_dates`;
CREATE TABLE `pso_data_validator`.`dvt_tricky_dates` (
CREATE OR REPLACE TABLE `pso_data_validator`.`dvt_tricky_dates` (
id INT64 NOT NULL
, col_dt_low DATE
, col_dt_epoch DATE
Expand Down
2 changes: 1 addition & 1 deletion tests/resources/mysql_test_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ CREATE TABLE `pso_data_validator`.`dvt_core_types`
, col_float64 double
, col_varchar_30 varchar(30)
, col_char_2 char(2)
, col_string varchar(21000)
, col_string text
, col_date date
, col_datetime datetime(3)
, col_tstz timestamp(3)
Expand Down
20 changes: 10 additions & 10 deletions tests/resources/postgresql_test_tables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
-- limitations under the License.

CREATE SCHEMA pso_data_validator;
DROP TABLE IF EXISTS pso_data_validator.dvt_core_types;
DROP TABLE IF EXISTS pso_data_validator.dvt_core_types CASCADE;
CREATE TABLE pso_data_validator.dvt_core_types
( id int NOT NULL PRIMARY KEY
, col_int8 smallint
Expand Down Expand Up @@ -51,7 +51,7 @@ INSERT INTO pso_data_validator.dvt_core_types VALUES
,DATE'1970-01-03',TIMESTAMP'1970-01-03 00:00:03'
,TIMESTAMP WITH TIME ZONE'1970-01-03 00:00:03 -03:00');

CREATE VIEW pso_data_validator.dvt_core_types_vw AS
CREATE OR REPLACE VIEW pso_data_validator.dvt_core_types_vw AS
SELECT * FROM pso_data_validator.dvt_core_types;

DROP TABLE IF EXISTS pso_data_validator.dvt_ora2pg_types;
Expand Down Expand Up @@ -215,7 +215,7 @@ CREATE TABLE pso_data_validator.dvt_pg_types
);
COMMENT ON TABLE pso_data_validator.dvt_pg_types IS 'PostgreSQL data types integration test table';

CREATE EXTENSION pgcrypto;
CREATE EXTENSION IF NOT EXISTS pgcrypto;
INSERT INTO pso_data_validator.dvt_pg_types
(col_int2,col_int4,col_int8,col_dec,col_dec_10_2
--,col_money
Expand Down Expand Up @@ -311,7 +311,7 @@ INSERT INTO pso_data_validator.dvt_binary VALUES
(CAST('DVT-key-4' AS bytea), 4, 'Row 4'),
(CAST('DVT-key-5' AS bytea), 5, 'Row 5');

DROP TABLE pso_data_validator.dvt_char_id;
DROP TABLE IF EXISTS pso_data_validator.dvt_char_id;
CREATE TABLE pso_data_validator.dvt_char_id
( id char(6) NOT NULL PRIMARY KEY
, other_data varchar(100)
Expand All @@ -324,7 +324,7 @@ INSERT INTO pso_data_validator.dvt_char_id VALUES
('DVT4', 'Row 4'),
('DVT5', 'Row 5');

DROP TABLE pso_data_validator.dvt_pangrams;
DROP TABLE IF EXISTS pso_data_validator.dvt_pangrams;
CREATE TABLE pso_data_validator.dvt_pangrams
( id int
, lang varchar(100)
Expand All @@ -346,7 +346,7 @@ INSERT INTO pso_data_validator.dvt_pangrams VALUES
(5,'Turkish', 'Pijamalı hasta yağız şoföre çabucak güvendi',
'The sick person in pyjamas quickly trusted the swarthy driver');

DROP TABLE pso_data_validator.dvt_many_cols;
DROP TABLE IF EXISTS pso_data_validator.dvt_many_cols;
CREATE TABLE pso_data_validator.dvt_many_cols
( id decimal(5)
, col_001 varchar(2)
Expand Down Expand Up @@ -752,7 +752,7 @@ CREATE TABLE pso_data_validator.dvt_many_cols
COMMENT ON TABLE pso_data_validator.dvt_many_cols IS 'Integration test table used to test validating many columns.';
INSERT INTO pso_data_validator.dvt_many_cols (id) values (1);

DROP TABLE pso_data_validator."dvt-identifier$_#";
DROP TABLE IF EXISTS pso_data_validator."dvt-identifier$_#";
CREATE TABLE pso_data_validator."dvt-identifier$_#"
( id int NOT NULL PRIMARY KEY
, "col#hash" varchar(10)
Expand All @@ -768,7 +768,7 @@ INSERT INTO pso_data_validator."dvt-identifier$_#" VALUES (3,'#','$','-','@','Ro
INSERT INTO pso_data_validator."dvt-identifier$_#" VALUES (4,'#','$','-','@','Row 4');
INSERT INTO pso_data_validator."dvt-identifier$_#" VALUES (5,'#','$','-','@','Row 5');

DROP TABLE pso_data_validator.dvt_bool;
DROP TABLE IF EXISTS pso_data_validator.dvt_bool;
CREATE TABLE pso_data_validator.dvt_bool
( id int NOT NULL PRIMARY KEY
, col_bool_dec boolean
Expand All @@ -779,7 +779,7 @@ COMMENT ON TABLE pso_data_validator.dvt_bool IS 'Integration test table used to
INSERT INTO pso_data_validator.dvt_bool VALUES (1,true,true,true,true);
INSERT INTO pso_data_validator.dvt_bool VALUES (2,false,false,false,false);

DROP TABLE pso_data_validator.dvt_uuid_id;
DROP TABLE IF EXISTS pso_data_validator.dvt_uuid_id;
CREATE TABLE pso_data_validator.dvt_uuid_id
( id uuid NOT NULL PRIMARY KEY
, col_uuid uuid
Expand All @@ -789,7 +789,7 @@ INSERT INTO pso_data_validator.dvt_uuid_id VALUES
(uuid('387bdc3b218443b28ec23ac791c5b0f1'),uuid('387bdc3b218443b28ec23ac791c5b0f1'),'A'),
(uuid('397bdc3b218443b28ec23ac791c5b0f1'),uuid('397bdc3b218443b28ec23ac791c5b0f1'),'B');

DROP TABLE pso_data_validator.dvt_group_by_timestamp;
DROP TABLE IF EXISTS pso_data_validator.dvt_group_by_timestamp;
CREATE TABLE pso_data_validator.dvt_group_by_timestamp
( id int NOT NULL PRIMARY KEY
, group_id int
Expand Down