Use segmentwise recompression in compress policy

Change compression policy to use segmentwise
recompression when possible to increase performance.
Segmentwise recompression decompresses rows into memory,
thus reducing IO load when recompressing, making it
much faster for bigger chunks.

.unreleased/enhancement_6343

@@ -0,0 +1,2 @@
+Implements: #6343 Enable segmentwise recompression in compression policy
+Thanks @fetchezar for reporting the issue

sql/policy_internal.sql

@@ -123,26 +123,19 @@ BEGIN
         )
       ) AND recompress_enabled IS TRUE THEN
       BEGIN
-        PERFORM @[email protected]_chunk(chunk_rec.oid, if_compressed => true);
+        -- first check if there's an index. Might have to use a heuristic to determine if index usage would be efficient,
+        -- or if we'd better fall back to decompressing & recompressing entire chunk
+        IF _timescaledb_functions.get_compressed_chunk_index_for_recompression(chunk_rec.oid) IS NOT NULL THEN
+          PERFORM _timescaledb_functions.recompress_chunk_segmentwise(chunk_rec.oid);
+        ELSE
+          PERFORM @[email protected]_chunk(chunk_rec.oid, if_compressed => true);
+          PERFORM @[email protected]_chunk(chunk_rec.oid);
+        END IF;
       EXCEPTION WHEN OTHERS THEN
         GET STACKED DIAGNOSTICS
             _message = MESSAGE_TEXT,
             _detail = PG_EXCEPTION_DETAIL;
-        RAISE WARNING 'decompressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text
-            USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail),
-                  ERRCODE = sqlstate;
-      END;
-      -- SET LOCAL is only active until end of transaction.
-      -- While we could use SET at the start of the function we do not
-      -- want to bleed out search_path to caller, so we do SET LOCAL
-      -- again after COMMIT
-      BEGIN
-        PERFORM @[email protected]_chunk(chunk_rec.oid);
-      EXCEPTION WHEN OTHERS THEN
-        GET STACKED DIAGNOSTICS
-            _message = MESSAGE_TEXT,
-            _detail = PG_EXCEPTION_DETAIL;
-        RAISE WARNING 'compressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text
+        RAISE WARNING 'recompressing chunk "%" failed when compression policy is executed', chunk_rec.oid::regclass::text
             USING DETAIL = format('Message: (%s), Detail: (%s).', _message, _detail),
                   ERRCODE = sqlstate;
       END;

tsl/test/expected/bgw_custom-15.out

@@ -572,7 +572,7 @@ FROM _timescaledb_config.bgw_job WHERE id = :job_id_5;
 
 --verify that job is dropped when cagg is dropped
 DROP MATERIALIZED VIEW conditions_summary_daily;
-NOTICE:  drop cascades to table _timescaledb_internal._hyper_3_10_chunk
+NOTICE:  drop cascades to table _timescaledb_internal._hyper_3_7_chunk
 SELECT id, proc_name, hypertable_id
 FROM _timescaledb_config.bgw_job WHERE id = :job_id_5;
  id | proc_name | hypertable_id 

tsl/test/expected/compression_bgw-13.out

@@ -474,39 +474,44 @@ ORDER BY 1, 2;
  Thu Apr 02 17:00:00 2020 PDT | 13 |     1
 (4 rows)
 
---chunk status should be unordered for the previously compressed chunk
+--chunk status should be partially compressed for the previously compressed chunk
 SELECT chunk_status,
-       chunk_name as "CHUNK_NAME"
+       chunk_name as "CHUNK_NAME",
+       compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
 FROM compressed_chunk_info_view
 WHERE hypertable_name = 'test2' ORDER BY chunk_name;
- chunk_status |     CHUNK_NAME     
---------------+--------------------
-            9 | _hyper_14_62_chunk
-            0 | _hyper_14_64_chunk
+ chunk_status |     CHUNK_NAME     |   COMPRESSED_CHUNK_NAME    
+--------------+--------------------+----------------------------
+            9 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
+            0 | _hyper_14_64_chunk | 
 (2 rows)
 
 SELECT add_compression_policy AS job_id
   FROM add_compression_policy('test2', '30d'::interval) \gset
 CALL run_job(:job_id);
 CALL run_job(:job_id);
 -- status should be compressed ---
+-- compressed chunk name should not change for
+-- the partially compressed chunk indicating 
+-- it was done segmentwise
 SELECT chunk_status,
-       chunk_name as "CHUNK_NAME"
+       chunk_name as "CHUNK_NAME",
+       compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
 FROM compressed_chunk_info_view
 WHERE hypertable_name = 'test2' ORDER BY chunk_name;
- chunk_status |     CHUNK_NAME     
---------------+--------------------
-            1 | _hyper_14_62_chunk
-            1 | _hyper_14_64_chunk
+ chunk_status |     CHUNK_NAME     |   COMPRESSED_CHUNK_NAME    
+--------------+--------------------+----------------------------
+            1 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
+            1 | _hyper_14_64_chunk | compress_hyper_15_65_chunk
 (2 rows)
 
 \set ON_ERROR_STOP 0
 -- call recompress_chunk when status is not unordered
 CALL recompress_chunk(:'CHUNK_NAME'::regclass, true);
-psql:include/recompress_basic.sql:110: NOTICE:  nothing to recompress in chunk "_hyper_14_62_chunk"
+psql:include/recompress_basic.sql:115: NOTICE:  nothing to recompress in chunk "_hyper_14_62_chunk"
 -- This will succeed and compress the chunk for the test below.
 CALL recompress_chunk(:'CHUNK_NAME'::regclass, false);
-psql:include/recompress_basic.sql:113: ERROR:  nothing to recompress in chunk "_hyper_14_62_chunk"
+psql:include/recompress_basic.sql:118: ERROR:  nothing to recompress in chunk "_hyper_14_62_chunk"
 --now decompress it , then try and recompress
 SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
              decompress_chunk             
@@ -515,7 +520,7 @@ SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
 (1 row)
 
 CALL recompress_chunk(:'CHUNK_NAME'::regclass);
-psql:include/recompress_basic.sql:117: ERROR:  call compress_chunk instead of recompress_chunk
+psql:include/recompress_basic.sql:122: ERROR:  call compress_chunk instead of recompress_chunk
 \set ON_ERROR_STOP 1
 -- test recompress policy
 CREATE TABLE metrics(time timestamptz NOT NULL);
@@ -620,7 +625,7 @@ SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'met
 
 ---- nothing to do yet
 CALL run_job(:JOB_RECOMPRESS);
-psql:include/recompress_basic.sql:189: NOTICE:  no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
+psql:include/recompress_basic.sql:194: NOTICE:  no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
 ---- status should be 1
 SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
  chunk_status 

tsl/test/expected/compression_bgw-14.out

@@ -474,39 +474,44 @@ ORDER BY 1, 2;
  Thu Apr 02 17:00:00 2020 PDT | 13 |     1
 (4 rows)
 
---chunk status should be unordered for the previously compressed chunk
+--chunk status should be partially compressed for the previously compressed chunk
 SELECT chunk_status,
-       chunk_name as "CHUNK_NAME"
+       chunk_name as "CHUNK_NAME",
+       compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
 FROM compressed_chunk_info_view
 WHERE hypertable_name = 'test2' ORDER BY chunk_name;
- chunk_status |     CHUNK_NAME     
---------------+--------------------
-            9 | _hyper_14_62_chunk
-            0 | _hyper_14_64_chunk
+ chunk_status |     CHUNK_NAME     |   COMPRESSED_CHUNK_NAME    
+--------------+--------------------+----------------------------
+            9 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
+            0 | _hyper_14_64_chunk | 
 (2 rows)
 
 SELECT add_compression_policy AS job_id
   FROM add_compression_policy('test2', '30d'::interval) \gset
 CALL run_job(:job_id);
 CALL run_job(:job_id);
 -- status should be compressed ---
+-- compressed chunk name should not change for
+-- the partially compressed chunk indicating 
+-- it was done segmentwise
 SELECT chunk_status,
-       chunk_name as "CHUNK_NAME"
+       chunk_name as "CHUNK_NAME",
+       compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
 FROM compressed_chunk_info_view
 WHERE hypertable_name = 'test2' ORDER BY chunk_name;
- chunk_status |     CHUNK_NAME     
---------------+--------------------
-            1 | _hyper_14_62_chunk
-            1 | _hyper_14_64_chunk
+ chunk_status |     CHUNK_NAME     |   COMPRESSED_CHUNK_NAME    
+--------------+--------------------+----------------------------
+            1 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
+            1 | _hyper_14_64_chunk | compress_hyper_15_65_chunk
 (2 rows)
 
 \set ON_ERROR_STOP 0
 -- call recompress_chunk when status is not unordered
 CALL recompress_chunk(:'CHUNK_NAME'::regclass, true);
-psql:include/recompress_basic.sql:110: NOTICE:  nothing to recompress in chunk "_hyper_14_62_chunk"
+psql:include/recompress_basic.sql:115: NOTICE:  nothing to recompress in chunk "_hyper_14_62_chunk"
 -- This will succeed and compress the chunk for the test below.
 CALL recompress_chunk(:'CHUNK_NAME'::regclass, false);
-psql:include/recompress_basic.sql:113: ERROR:  nothing to recompress in chunk "_hyper_14_62_chunk"
+psql:include/recompress_basic.sql:118: ERROR:  nothing to recompress in chunk "_hyper_14_62_chunk"
 --now decompress it , then try and recompress
 SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
              decompress_chunk             
@@ -515,7 +520,7 @@ SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
 (1 row)
 
 CALL recompress_chunk(:'CHUNK_NAME'::regclass);
-psql:include/recompress_basic.sql:117: ERROR:  call compress_chunk instead of recompress_chunk
+psql:include/recompress_basic.sql:122: ERROR:  call compress_chunk instead of recompress_chunk
 \set ON_ERROR_STOP 1
 -- test recompress policy
 CREATE TABLE metrics(time timestamptz NOT NULL);
@@ -620,7 +625,7 @@ SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'met
 
 ---- nothing to do yet
 CALL run_job(:JOB_RECOMPRESS);
-psql:include/recompress_basic.sql:189: NOTICE:  no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
+psql:include/recompress_basic.sql:194: NOTICE:  no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
 ---- status should be 1
 SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
  chunk_status 

tsl/test/expected/compression_bgw-15.out

@@ -474,39 +474,44 @@ ORDER BY 1, 2;
  Thu Apr 02 17:00:00 2020 PDT | 13 |     1
 (4 rows)
 
---chunk status should be unordered for the previously compressed chunk
+--chunk status should be partially compressed for the previously compressed chunk
 SELECT chunk_status,
-       chunk_name as "CHUNK_NAME"
+       chunk_name as "CHUNK_NAME",
+       compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
 FROM compressed_chunk_info_view
 WHERE hypertable_name = 'test2' ORDER BY chunk_name;
- chunk_status |     CHUNK_NAME     
---------------+--------------------
-            9 | _hyper_14_62_chunk
-            0 | _hyper_14_64_chunk
+ chunk_status |     CHUNK_NAME     |   COMPRESSED_CHUNK_NAME    
+--------------+--------------------+----------------------------
+            9 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
+            0 | _hyper_14_64_chunk | 
 (2 rows)
 
 SELECT add_compression_policy AS job_id
   FROM add_compression_policy('test2', '30d'::interval) \gset
 CALL run_job(:job_id);
 CALL run_job(:job_id);
 -- status should be compressed ---
+-- compressed chunk name should not change for
+-- the partially compressed chunk indicating 
+-- it was done segmentwise
 SELECT chunk_status,
-       chunk_name as "CHUNK_NAME"
+       chunk_name as "CHUNK_NAME",
+       compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
 FROM compressed_chunk_info_view
 WHERE hypertable_name = 'test2' ORDER BY chunk_name;
- chunk_status |     CHUNK_NAME     
---------------+--------------------
-            1 | _hyper_14_62_chunk
-            1 | _hyper_14_64_chunk
+ chunk_status |     CHUNK_NAME     |   COMPRESSED_CHUNK_NAME    
+--------------+--------------------+----------------------------
+            1 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
+            1 | _hyper_14_64_chunk | compress_hyper_15_65_chunk
 (2 rows)
 
 \set ON_ERROR_STOP 0
 -- call recompress_chunk when status is not unordered
 CALL recompress_chunk(:'CHUNK_NAME'::regclass, true);
-psql:include/recompress_basic.sql:110: NOTICE:  nothing to recompress in chunk "_hyper_14_62_chunk"
+psql:include/recompress_basic.sql:115: NOTICE:  nothing to recompress in chunk "_hyper_14_62_chunk"
 -- This will succeed and compress the chunk for the test below.
 CALL recompress_chunk(:'CHUNK_NAME'::regclass, false);
-psql:include/recompress_basic.sql:113: ERROR:  nothing to recompress in chunk "_hyper_14_62_chunk"
+psql:include/recompress_basic.sql:118: ERROR:  nothing to recompress in chunk "_hyper_14_62_chunk"
 --now decompress it , then try and recompress
 SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
              decompress_chunk             
@@ -515,7 +520,7 @@ SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
 (1 row)
 
 CALL recompress_chunk(:'CHUNK_NAME'::regclass);
-psql:include/recompress_basic.sql:117: ERROR:  call compress_chunk instead of recompress_chunk
+psql:include/recompress_basic.sql:122: ERROR:  call compress_chunk instead of recompress_chunk
 \set ON_ERROR_STOP 1
 -- test recompress policy
 CREATE TABLE metrics(time timestamptz NOT NULL);
@@ -620,7 +625,7 @@ SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'met
 
 ---- nothing to do yet
 CALL run_job(:JOB_RECOMPRESS);
-psql:include/recompress_basic.sql:189: NOTICE:  no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
+psql:include/recompress_basic.sql:194: NOTICE:  no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
 ---- status should be 1
 SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
  chunk_status 

tsl/test/expected/compression_bgw-16.out

@@ -474,39 +474,44 @@ ORDER BY 1, 2;
  Thu Apr 02 17:00:00 2020 PDT | 13 |     1
 (4 rows)
 
---chunk status should be unordered for the previously compressed chunk
+--chunk status should be partially compressed for the previously compressed chunk
 SELECT chunk_status,
-       chunk_name as "CHUNK_NAME"
+       chunk_name as "CHUNK_NAME",
+       compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
 FROM compressed_chunk_info_view
 WHERE hypertable_name = 'test2' ORDER BY chunk_name;
- chunk_status |     CHUNK_NAME     
---------------+--------------------
-            9 | _hyper_14_62_chunk
-            0 | _hyper_14_64_chunk
+ chunk_status |     CHUNK_NAME     |   COMPRESSED_CHUNK_NAME    
+--------------+--------------------+----------------------------
+            9 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
+            0 | _hyper_14_64_chunk | 
 (2 rows)
 
 SELECT add_compression_policy AS job_id
   FROM add_compression_policy('test2', '30d'::interval) \gset
 CALL run_job(:job_id);
 CALL run_job(:job_id);
 -- status should be compressed ---
+-- compressed chunk name should not change for
+-- the partially compressed chunk indicating 
+-- it was done segmentwise
 SELECT chunk_status,
-       chunk_name as "CHUNK_NAME"
+       chunk_name as "CHUNK_NAME",
+       compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
 FROM compressed_chunk_info_view
 WHERE hypertable_name = 'test2' ORDER BY chunk_name;
- chunk_status |     CHUNK_NAME     
---------------+--------------------
-            1 | _hyper_14_62_chunk
-            1 | _hyper_14_64_chunk
+ chunk_status |     CHUNK_NAME     |   COMPRESSED_CHUNK_NAME    
+--------------+--------------------+----------------------------
+            1 | _hyper_14_62_chunk | compress_hyper_15_63_chunk
+            1 | _hyper_14_64_chunk | compress_hyper_15_65_chunk
 (2 rows)
 
 \set ON_ERROR_STOP 0
 -- call recompress_chunk when status is not unordered
 CALL recompress_chunk(:'CHUNK_NAME'::regclass, true);
-psql:include/recompress_basic.sql:110: NOTICE:  nothing to recompress in chunk "_hyper_14_62_chunk"
+psql:include/recompress_basic.sql:115: NOTICE:  nothing to recompress in chunk "_hyper_14_62_chunk"
 -- This will succeed and compress the chunk for the test below.
 CALL recompress_chunk(:'CHUNK_NAME'::regclass, false);
-psql:include/recompress_basic.sql:113: ERROR:  nothing to recompress in chunk "_hyper_14_62_chunk"
+psql:include/recompress_basic.sql:118: ERROR:  nothing to recompress in chunk "_hyper_14_62_chunk"
 --now decompress it , then try and recompress
 SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
              decompress_chunk             
@@ -515,7 +520,7 @@ SELECT decompress_chunk(:'CHUNK_NAME'::regclass);
 (1 row)
 
 CALL recompress_chunk(:'CHUNK_NAME'::regclass);
-psql:include/recompress_basic.sql:117: ERROR:  call compress_chunk instead of recompress_chunk
+psql:include/recompress_basic.sql:122: ERROR:  call compress_chunk instead of recompress_chunk
 \set ON_ERROR_STOP 1
 -- test recompress policy
 CREATE TABLE metrics(time timestamptz NOT NULL);
@@ -620,7 +625,7 @@ SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'met
 
 ---- nothing to do yet
 CALL run_job(:JOB_RECOMPRESS);
-psql:include/recompress_basic.sql:189: NOTICE:  no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
+psql:include/recompress_basic.sql:194: NOTICE:  no chunks for hypertable "public.metrics" that satisfy recompress chunk policy
 ---- status should be 1
 SELECT chunk_status FROM compressed_chunk_info_view WHERE hypertable_name = 'metrics';
  chunk_status 

tsl/test/sql/include/recompress_basic.sql

@@ -88,9 +88,10 @@ FROM test2
 GROUP BY time_bucket(INTERVAL '2 hour', timec), b
 ORDER BY 1, 2;
 
---chunk status should be unordered for the previously compressed chunk
+--chunk status should be partially compressed for the previously compressed chunk
 SELECT chunk_status,
-       chunk_name as "CHUNK_NAME"
+       chunk_name as "CHUNK_NAME",
+       compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
 FROM compressed_chunk_info_view
 WHERE hypertable_name = 'test2' ORDER BY chunk_name;
 
@@ -100,8 +101,12 @@ CALL run_job(:job_id);
 CALL run_job(:job_id);
 
 -- status should be compressed ---
+-- compressed chunk name should not change for
+-- the partially compressed chunk indicating 
+-- it was done segmentwise
 SELECT chunk_status,
-       chunk_name as "CHUNK_NAME"
+       chunk_name as "CHUNK_NAME",
+       compressed_chunk_name as "COMPRESSED_CHUNK_NAME"
 FROM compressed_chunk_info_view
 WHERE hypertable_name = 'test2' ORDER BY chunk_name;