diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_schema_evolution.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_schema_evolution.q index 939904a8cba12..3c822db3a120c 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_schema_evolution.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_major_compaction_schema_evolution.q @@ -16,6 +16,8 @@ --! qt:replace:/(MAJOR\s+succeeded\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ -- Mask compaction id as they will be allocated in parallel threads --! qt:replace:/^[0-9]/#Masked#/ +-- Mask removed file size +--! qt:replace:/(\S\"removed-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ set hive.llap.io.enabled=true; set hive.vectorized.execution.enabled=true; diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out index f2f453b34d097..90d5541ca61b5 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_major_compaction_schema_evolution.q.out @@ -227,7 +227,7 @@ Table Parameters: bucketing_version 2 current-schema {\"type\":\"struct\",\"schema-id\":2,\"fields\":[{\"id\":1,\"name\":\"fname\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"address\",\"required\":false,\"type\":\"string\"}]} current-snapshot-id #Masked# - current-snapshot-summary {\"deleted-data-files\":\"6\",\"deleted-records\":\"6\",\"removed-files-size\":\"3167\",\"changed-partition-count\":\"2\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"10\",\"total-delete-files\":\"8\",\"total-position-deletes\":\"8\",\"total-equality-deletes\":\"0\"} + current-snapshot-summary {\"deleted-data-files\":\"6\",\"deleted-records\":\"6\",\"removed-files-size\":\"#Masked#\",\"changed-partition-count\":\"2\",\"total-records\":\"10\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"10\",\"total-delete-files\":\"8\",\"total-position-deletes\":\"8\",\"total-equality-deletes\":\"0\"} current-snapshot-timestamp-ms #Masked# default-partition-spec {\"spec-id\":0,\"fields\":[{\"name\":\"dept_id\",\"transform\":\"identity\",\"source-id\":3,\"field-id\":1000}]} format-version 2 diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java index 26302e6c143ce..3ec688562a910 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java @@ -881,7 +881,7 @@ public void testStatsAfterQueryCompactionOnTez() throws Exception { parameters = Hive.get().getTable(tblName).getParameters(); Assert.assertEquals("The number of files is differing from the expected", "1", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "736", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "735", parameters.get("totalSize")); } @Test diff --git a/pom.xml b/pom.xml index 2a63a4f0c237c..d7e1fe1a040bc 100644 --- a/pom.xml +++ b/pom.xml @@ -180,8 +180,8 @@ 8.0.31 42.7.3 21.3.0.0 - 2.3 - 1.8.5 + 5.9 + 1.9.3 3.4.4 4.11.0 2.0.0-M5 diff --git a/ql/pom.xml b/ql/pom.xml index 5fc3185411b6c..9004c3f015e37 100644 --- a/ql/pom.xml +++ b/ql/pom.xml @@ -536,11 +536,6 @@ stax stax-api - - net.sf.opencsv - opencsv - ${opencsv.version} - org.apache.hive hive-standalone-metastore-server @@ -759,6 +754,17 @@ + + com.opencsv + opencsv + ${opencsv.version} + + + commons-beanutils + commons-beanutils + + + org.apache.tez tez-dag diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java index 7409179f43682..7a204accbe49b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnNoBuckets.java @@ -790,7 +790,7 @@ public void testCompactStatsGather() throws Exception { .getParameters(); Assert.assertEquals("The number of files is differing from the expected", "1", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "2", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "692", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "693", parameters.get("totalSize")); int[][] targetVals2 = {{5, 1, 1}, {5, 2, 2}, {5, 3, 1}, {5, 4, 2}}; runStatementOnDriver("insert into T partition(p=1,q) " + makeValuesClause(targetVals2)); @@ -844,7 +844,7 @@ public void testCompactStatsGather() throws Exception { .getParameters(); Assert.assertEquals("The number of files is differing from the expected", "1", parameters.get("numFiles")); Assert.assertEquals("The number of rows is differing from the expected", "4", parameters.get("numRows")); - Assert.assertEquals("The total table size is differing from the expected", "705", parameters.get("totalSize")); + Assert.assertEquals("The total table size is differing from the expected", "704", parameters.get("totalSize")); } @Test diff --git a/ql/src/test/queries/clientpositive/insert_values_orig_table_use_metadata.q b/ql/src/test/queries/clientpositive/insert_values_orig_table_use_metadata.q index ff7c1a83556e8..556771a42f463 100644 --- a/ql/src/test/queries/clientpositive/insert_values_orig_table_use_metadata.q +++ b/ql/src/test/queries/clientpositive/insert_values_orig_table_use_metadata.q @@ -1,5 +1,6 @@ --! qt:dataset:src --! qt:dataset:alltypesorc +--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/ set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat; diff --git a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_10.q b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_10.q index 417872c272722..ff99561a8cd15 100644 --- a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_10.q +++ b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_10.q @@ -1,6 +1,6 @@ -- Try to run incremental on a non-transactional MV in presence of delete operations -- Compiler should fall back to full rebuild. - +--! qt:replace:/(\S Data size\:\s+)\S+(\s+Basic stats\: \S+ Column stats\: \S+)/$1#Masked#$2/ set hive.support.concurrency=true; set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; diff --git a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_5.q b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_5.q index cdaf1abd091ba..758003b3fd8fe 100644 --- a/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_5.q +++ b/ql/src/test/queries/clientpositive/materialized_view_create_rewrite_5.q @@ -1,3 +1,4 @@ +--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/ -- Test Incremental rebuild of materialized view without aggregate when source tables have -- 1) insert operations only -- 2) update/delete operations since last rebuild. diff --git a/ql/src/test/queries/clientpositive/orc_llap_nonvector.q b/ql/src/test/queries/clientpositive/orc_llap_nonvector.q index 95a0384ccaf7d..31a3d63c58acf 100644 --- a/ql/src/test/queries/clientpositive/orc_llap_nonvector.q +++ b/ql/src/test/queries/clientpositive/orc_llap_nonvector.q @@ -1,4 +1,5 @@ --! qt:dataset:alltypesorc +--! qt:replace:/(\S Data size\:\s+)\S+(\s+Basic stats\: \S+ Column stats\: \S+)/$1#Masked#$2/ set hive.vectorized.execution.enabled=false; set hive.mapred.mode=nonstrict; diff --git a/ql/src/test/queries/clientpositive/orc_merge12.q b/ql/src/test/queries/clientpositive/orc_merge12.q index 348c3a8f5e086..d1487d2209853 100644 --- a/ql/src/test/queries/clientpositive/orc_merge12.q +++ b/ql/src/test/queries/clientpositive/orc_merge12.q @@ -1,4 +1,5 @@ --! qt:replace:/(File Version:)(.+)/$1#Masked#/ +--! qt:replace:/(File length:\s+)\S+(\s+bytes)/$1#Masked#$2/ set hive.vectorized.execution.enabled=false; CREATE TABLE `alltypesorc3xcols`( diff --git a/ql/src/test/queries/clientpositive/stats_part.q b/ql/src/test/queries/clientpositive/stats_part.q index d0812e1007814..99a8a458ccb7a 100644 --- a/ql/src/test/queries/clientpositive/stats_part.q +++ b/ql/src/test/queries/clientpositive/stats_part.q @@ -1,3 +1,5 @@ +-- Mask the totalSize value as it can have slight variability, causing test flakiness +--! qt:replace:/(\s+totalSize\s+)\S+(\s+)/$1#Masked#$2/ set hive.stats.dbclass=fs; set hive.stats.fetch.column.stats=true; set datanucleus.cache.collections=false; diff --git a/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out b/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out index cc838125f2213..5a71fa859f903 100644 --- a/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out +++ b/ql/src/test/results/clientpositive/llap/insert_values_orig_table_use_metadata.q.out @@ -108,7 +108,7 @@ Table Parameters: numFiles 0 numRows 0 rawDataSize 0 - totalSize 0 + totalSize #Masked# transactional true transactional_properties default #### A masked pattern was here #### @@ -173,7 +173,7 @@ Table Parameters: numFiles 1 numRows 12288 rawDataSize 0 - totalSize 309572 + totalSize #Masked# transactional true transactional_properties default #### A masked pattern was here #### @@ -344,7 +344,7 @@ Table Parameters: numFiles 1 numRows 2 rawDataSize 0 - totalSize 1673 + totalSize #Masked# transactional true transactional_properties default #### A masked pattern was here #### @@ -440,7 +440,7 @@ Table Parameters: numFiles 2 numRows 4 rawDataSize 0 - totalSize 3346 + totalSize #Masked# transactional true transactional_properties default #### A masked pattern was here #### @@ -532,7 +532,7 @@ Table Parameters: numFiles 3 numRows 12292 rawDataSize 0 - totalSize 312919 + totalSize #Masked# transactional true transactional_properties default #### A masked pattern was here #### @@ -603,7 +603,7 @@ Table Parameters: numFiles 0 numRows 0 rawDataSize 0 - totalSize 0 + totalSize #Masked# #### A masked pattern was here #### # Storage Information @@ -648,7 +648,7 @@ Table Parameters: numFiles 1 numRows 500 rawDataSize 5312 - totalSize 5812 + totalSize #Masked# #### A masked pattern was here #### # Storage Information @@ -735,7 +735,7 @@ Partition Parameters: numFiles 1 numRows 0 rawDataSize 0 - totalSize 5812 + totalSize #Masked# #### A masked pattern was here #### # Storage Information @@ -860,7 +860,7 @@ Partition Parameters: numFiles 2 numRows 0 rawDataSize 0 - totalSize 5820 + totalSize #Masked# #### A masked pattern was here #### # Storage Information @@ -910,7 +910,7 @@ Partition Parameters: numFiles 2 numRows 502 rawDataSize 5318 - totalSize 5820 + totalSize #Masked# #### A masked pattern was here #### # Storage Information diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_10.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_10.q.out index 86cce82f092cf..5ff07b9fc54d9 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_10.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_10.q.out @@ -79,17 +79,17 @@ STAGE PLANS: TableScan alias: t1 filterExpr: (b = 1) (type: boolean) - Statistics: Num rows: 69 Data size: 13710 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 70 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Filter Operator predicate: (b = 1) (type: boolean) - Statistics: Num rows: 1 Data size: 198 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE Select Operator expressions: a (type: int), 1 (type: int) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 198 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 198 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: #Masked# Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out index 1b06d867b39cf..26b35f42a42e7 100644 --- a/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out +++ b/ql/src/test/results/clientpositive/llap/materialized_view_create_rewrite_5.q.out @@ -404,7 +404,7 @@ Table Parameters: numFiles 2 numRows 10 rawDataSize 0 - totalSize 1522 + totalSize #Masked# transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out index 669df679d82c1..8ef0f0dde97ae 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_nonvector.q.out @@ -94,17 +94,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: orc_llap_nonvector - Statistics: Num rows: 12288 Data size: 2942394 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12288 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 100 - Statistics: Num rows: 100 Data size: 24360 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 100 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ctinyint (type: tinyint), csmallint (type: smallint), cint (type: int), cbigint (type: bigint), cfloat (type: float), cdouble (type: double), cstring1 (type: string), cstring2 (type: string), ctimestamp1 (type: timestamp), ctimestamp2 (type: timestamp), cboolean1 (type: boolean), cboolean2 (type: boolean), rdm (type: double) outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12 - Statistics: Num rows: 100 Data size: 24360 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 100 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 100 Data size: 24360 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 100 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -249,17 +249,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: orc_llap_nonvector - Statistics: Num rows: 12288 Data size: 899146 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12288 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 1025 - Statistics: Num rows: 1025 Data size: 75068 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1025 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: cint (type: int), cstring1 (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1025 Data size: 75068 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1025 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 1025 Data size: 75068 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1025 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat @@ -1354,17 +1354,17 @@ STAGE PLANS: Map Operator Tree: TableScan alias: orc_llap_nonvector_2 - Statistics: Num rows: 12288 Data size: 4468250 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12288 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Limit Number of rows: 10 - Statistics: Num rows: 10 Data size: 80 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: ROW__ID (type: struct) outputColumnNames: _col0 - Statistics: Num rows: 10 Data size: 760 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 10 Data size: 760 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: #Masked# Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.SequenceFileInputFormat output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat diff --git a/ql/src/test/results/clientpositive/llap/stats_part.q.out b/ql/src/test/results/clientpositive/llap/stats_part.q.out index d9a4eb513f6ed..ed9117d6757a8 100644 --- a/ql/src/test/results/clientpositive/llap/stats_part.q.out +++ b/ql/src/test/results/clientpositive/llap/stats_part.q.out @@ -161,7 +161,7 @@ Table Parameters: numPartitions 0 numRows 0 rawDataSize 0 - totalSize 0 + totalSize #Masked# transactional true transactional_properties default #### A masked pattern was here #### @@ -231,7 +231,7 @@ Table Parameters: numPartitions 3 numRows 6 rawDataSize 0 - totalSize 2292 + totalSize #Masked# transactional true transactional_properties default #### A masked pattern was here #### @@ -292,7 +292,7 @@ Table Parameters: numPartitions 3 numRows 8 rawDataSize 0 - totalSize 3066 + totalSize #Masked# transactional true transactional_properties default #### A masked pattern was here #### @@ -449,7 +449,7 @@ Table Parameters: numPartitions 3 numRows 8 rawDataSize 0 - totalSize 3066 + totalSize #Masked# transactional true transactional_properties default #### A masked pattern was here #### @@ -534,7 +534,7 @@ Table Parameters: numPartitions 3 numRows 8 rawDataSize 0 - totalSize 3066 + totalSize #Masked# transactional true transactional_properties default #### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out b/ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out index 39133ac7b1263..3c83b5608fd97 100644 --- a/ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out +++ b/ql/src/test/results/clientpositive/llap/vector_decimal_aggregate.q.out @@ -403,7 +403,7 @@ POSTHOOK: Input: default@decimal_vgby 626923679 1024 9723.4027027027 -9778.9513513514 10541.0525297287 10.29399661106318 5742.091453237337 5744.897264034264 1024 11645.74615384615400 -11712.27692307692300 12625.04759999997746 12.329148046874977988 6877.318722794881 6880.679250101608 6981 3 5831542.2692483780 -515.6210729730 5830511.0271024320 1943503.67570081066667 2749258.455012492 3367140.1929065133 3 6984454.21109769200000 -617.56077692307690 6983219.08954384584620 2327739.696514615282066667 3292794.4113115156 4032833.0678006653 762 2 5831542.2692483780 1531.2194054054 5833073.4886537834 2916536.74432689170000 2915005.5249214866 4122440.3477364695 2 6984454.21109769200000 1833.94569230769250 6986288.15678999969250 3493144.078394999846250000 3491310.1327026924 4937458.140118757 -NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 1633.60810810806667 5695.483082135323 5696.410307714464 3072 11160.71538461538500 -5147.90769230769300 6010604.30769230735360 1956.576923076922966667 6821.495748565151 6822.606289190915 +NULL 3072 9318.4351351351 -4298.1513513514 5018444.1081079808 1633.60810810806667 5695.483082135325 5696.4103077144655 3072 11160.71538461538500 -5147.90769230769300 6010604.30769230735360 1956.576923076922966667 6821.4957485651385 6822.606289190904 PREHOOK: query: CREATE TABLE decimal_vgby_small STORED AS TEXTFILE AS SELECT cdouble, CAST (((cdouble*22.1)/37) AS DECIMAL(11,5)) AS cdecimal1, CAST (((cdouble*9.3)/13) AS DECIMAL(16,0)) AS cdecimal2, @@ -828,7 +828,7 @@ POSTHOOK: Input: default@decimal_vgby_small 626923679 1024 9723.40270 -9778.95135 10541.05247 10.293996553 5742.091453325365 5744.897264122335 1024 11646 -11712 12641 12.3447 6877.306686989158 6880.6672084147185 6981 2 -515.62107 -515.62107 -1031.24214 -515.621070000 0.0 0.0 3 6984454 -618 6983218 2327739.3333 3292794.518850853 4032833.1995089175 762 1 1531.21941 1531.21941 1531.21941 1531.219410000 0.0 NULL 2 6984454 1834 6986288 3493144.0000 3491310.0 4937457.95244881 -NULL 3072 9318.43514 -4298.15135 5018444.11392 1633.608110000 5695.4830839098695 5696.410309489299 3072 11161 -5148 6010880 1956.6667 6821.647911041892 6822.758476439734 +NULL 3072 9318.43514 -4298.15135 5018444.11392 1633.608110000 5695.483083909676 5696.410309489105 3072 11161 -5148 6010880 1956.6667 6821.647911041892 6822.758476439734 PREHOOK: query: SELECT SUM(HASH(*)) FROM (SELECT cint, COUNT(cdecimal1), MAX(cdecimal1), MIN(cdecimal1), SUM(cdecimal1), AVG(cdecimal1), STDDEV_POP(cdecimal1), STDDEV_SAMP(cdecimal1), @@ -847,4 +847,4 @@ FROM (SELECT cint, POSTHOOK: type: QUERY POSTHOOK: Input: default@decimal_vgby_small #### A masked pattern was here #### -95165244160 +95767761728 diff --git a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out index 3c6ddb257a4a3..894566a3bf18d 100644 --- a/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out +++ b/ql/src/test/results/clientpositive/tez/acid_vectorization_original_tez.q.out @@ -375,7 +375,7 @@ Found 4 items -rw-rw-rw- 3 ### USER ### ### GROUP ### 8753 ### HDFS DATE ### hdfs://### HDFS PATH ### -rw-rw-rw- 3 ### USER ### ### GROUP ### 7531 ### HDFS DATE ### hdfs://### HDFS PATH ### -rw-rw-rw- 3 ### USER ### ### GROUP ### 7174 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7066 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7065 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: insert into over10k_orc_bucketed_n0 select * from over10k_n9 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_n9 @@ -402,8 +402,8 @@ Found 8 items -rw-rw-rw- 3 ### USER ### ### GROUP ### 7531 ### HDFS DATE ### hdfs://### HDFS PATH ### -rw-rw-rw- 3 ### USER ### ### GROUP ### 7174 ### HDFS DATE ### hdfs://### HDFS PATH ### -rw-rw-rw- 3 ### USER ### ### GROUP ### 7174 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7066 ### HDFS DATE ### hdfs://### HDFS PATH ### --rw-rw-rw- 3 ### USER ### ### GROUP ### 7066 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7065 ### HDFS DATE ### hdfs://### HDFS PATH ### +-rw-rw-rw- 3 ### USER ### ### GROUP ### 7065 ### HDFS DATE ### hdfs://### HDFS PATH ### PREHOOK: query: select distinct 7 as seven, INPUT__FILE__NAME from over10k_orc_bucketed_n0 PREHOOK: type: QUERY PREHOOK: Input: default@over10k_orc_bucketed_n0 diff --git a/ql/src/test/results/clientpositive/tez/orc_merge12.q.out b/ql/src/test/results/clientpositive/tez/orc_merge12.q.out index 73526e4e93e23..3bda372791063 100644 --- a/ql/src/test/results/clientpositive/tez/orc_merge12.q.out +++ b/ql/src/test/results/clientpositive/tez/orc_merge12.q.out @@ -814,7 +814,7 @@ Stripes: Entry 0: count: 6889 hasNull: true true: 3402 positions: 0,0,0,0,0,0,0,0 Entry 1: count: 2284 hasNull: true true: 581 positions: 0,168,8,0,0,520,97,1 -File length: 3004637 bytes +File length: #Masked# bytes Padding length: 0 bytes Padding ratio: 0% ________________________________________________________________________________________________________________________ diff --git a/serde/pom.xml b/serde/pom.xml index b6c0f53edacef..ad0bbcb7f6010 100644 --- a/serde/pom.xml +++ b/serde/pom.xml @@ -100,7 +100,7 @@ - net.sf.opencsv + com.opencsv opencsv ${opencsv.version} diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/OpenCSVSerde.java b/serde/src/java/org/apache/hadoop/hive/serde2/OpenCSVSerde.java index 243eeacf77106..9bb2d887527fb 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/OpenCSVSerde.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/OpenCSVSerde.java @@ -39,8 +39,10 @@ import java.util.Objects; import java.util.Properties; -import au.com.bytecode.opencsv.CSVReader; -import au.com.bytecode.opencsv.CSVWriter; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.CSVWriter; /** * OpenCSVSerde use opencsv to deserialize CSV format. @@ -179,15 +181,17 @@ private CSVReader newReader(final Reader reader, char separator, char quote, cha // CSVReader will throw an exception if any of separator, quote, or escape is the same, but // the CSV format specifies that the escape character and quote char are the same... very weird if (CSVWriter.DEFAULT_ESCAPE_CHARACTER == escape) { - return new CSVReader(reader, separator, quote); + return new CSVReaderBuilder(reader).withCSVParser( + new CSVParserBuilder().withSeparator(separator).withQuoteChar(quote).build()).build(); } else { - return new CSVReader(reader, separator, quote, escape); + return new CSVReaderBuilder(reader).withCSVParser( + new CSVParserBuilder().withSeparator(separator).withQuoteChar(quote).withEscapeChar(escape).build()).build(); } } private CSVWriter newWriter(final Writer writer, char separator, char quote, char escape) { if (CSVWriter.DEFAULT_ESCAPE_CHARACTER == escape) { - return new CSVWriter(writer, separator, quote, ""); + return new CSVWriter(writer, separator, quote, '"', ""); } else { return new CSVWriter(writer, separator, quote, escape, ""); } diff --git a/standalone-metastore/pom.xml b/standalone-metastore/pom.xml index 0afdeae161963..1be3979a0785a 100644 --- a/standalone-metastore/pom.xml +++ b/standalone-metastore/pom.xml @@ -89,7 +89,7 @@ 0.16.0 2.18.0 3.4.4 - 1.8.5 + 1.9.3 3.24.4 1.51.0 1.9.0