Added telemetry for frozen tuples

sql/pre_install/tables.sql

@@ -495,6 +495,7 @@ CREATE TABLE _timescaledb_catalog.compression_chunk_size (
   compressed_index_size bigint NOT NULL,
   numrows_pre_compression bigint,
   numrows_post_compression bigint,
+  numrows_frozen bigint,
   -- table constraints
   CONSTRAINT compression_chunk_size_pkey PRIMARY KEY (chunk_id),
   CONSTRAINT compression_chunk_size_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE,

sql/updates/latest-dev.sql

@@ -179,3 +179,53 @@ DROP TABLE _timescaledb_internal.tmp_chunk_seq_value;
 GRANT SELECT ON _timescaledb_catalog.chunk_id_seq TO PUBLIC;
 GRANT SELECT ON _timescaledb_catalog.chunk TO PUBLIC;
 -- end recreate _timescaledb_catalog.chunk table --
+
+--
+-- Rebuild the catalog table `_timescaledb_catalog.compression_chunk_size` to
+-- add new column `numrows_frozen`
+--
+CREATE TABLE _timescaledb_internal.compression_chunk_size_tmp
+    AS SELECT * from _timescaledb_catalog.compression_chunk_size;
+
+-- Drop depended views
+-- We assume that '_timescaledb_internal.compressed_chunk_stats' was already dropped in this update
+-- (see above)
+
+-- Drop table
+ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.compression_chunk_size;
+DROP TABLE _timescaledb_catalog.compression_chunk_size;
+
+CREATE TABLE _timescaledb_catalog.compression_chunk_size (
+  chunk_id integer NOT NULL,
+  compressed_chunk_id integer NOT NULL,
+  uncompressed_heap_size bigint NOT NULL,
+  uncompressed_toast_size bigint NOT NULL,
+  uncompressed_index_size bigint NOT NULL,
+  compressed_heap_size bigint NOT NULL,
+  compressed_toast_size bigint NOT NULL,
+  compressed_index_size bigint NOT NULL,
+  numrows_pre_compression bigint,
+  numrows_post_compression bigint,
+  numrows_frozen bigint,
+  -- table constraints
+  CONSTRAINT compression_chunk_size_pkey PRIMARY KEY (chunk_id),
+  CONSTRAINT compression_chunk_size_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE,
+  CONSTRAINT compression_chunk_size_compressed_chunk_id_fkey FOREIGN KEY (compressed_chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE
+);
+
+INSERT INTO _timescaledb_catalog.compression_chunk_size
+(chunk_id, compressed_chunk_id, uncompressed_heap_size, uncompressed_toast_size,
+  uncompressed_index_size, compressed_heap_size, compressed_toast_size,
+  compressed_index_size, numrows_pre_compression, numrows_post_compression, numrows_frozen)
+SELECT chunk_id, compressed_chunk_id, uncompressed_heap_size, uncompressed_toast_size,
+  uncompressed_index_size, compressed_heap_size, compressed_toast_size,
+  compressed_index_size, numrows_pre_compression, numrows_post_compression, 0
+FROM _timescaledb_internal.compression_chunk_size_tmp;
+
+DROP TABLE _timescaledb_internal.compression_chunk_size_tmp;
+
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_chunk_size', '');
+
+GRANT SELECT ON _timescaledb_catalog.compression_chunk_size TO PUBLIC;
+
+-- End modify `_timescaledb_catalog.compression_chunk_size`

sql/updates/reverse-dev.sql

@@ -124,3 +124,53 @@ GRANT SELECT ON _timescaledb_catalog.chunk_id_seq TO PUBLIC;
 GRANT SELECT ON _timescaledb_catalog.chunk TO PUBLIC;
 
 -- end recreate _timescaledb_catalog.chunk table --
+
+
+--
+-- Rebuild the catalog table `_timescaledb_catalog.compression_chunk_size` to
+-- remove column `numrows_frozen`
+--
+CREATE TABLE _timescaledb_internal.compression_chunk_size_tmp
+    AS SELECT * from _timescaledb_catalog.compression_chunk_size;
+
+-- Drop depended views
+-- We assume that '_timescaledb_internal.compressed_chunk_stats' was already dropped in this update
+-- (see above)
+
+-- Drop table
+ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.compression_chunk_size;
+DROP TABLE _timescaledb_catalog.compression_chunk_size;
+
+CREATE TABLE _timescaledb_catalog.compression_chunk_size (
+  chunk_id integer NOT NULL,
+  compressed_chunk_id integer NOT NULL,
+  uncompressed_heap_size bigint NOT NULL,
+  uncompressed_toast_size bigint NOT NULL,
+  uncompressed_index_size bigint NOT NULL,
+  compressed_heap_size bigint NOT NULL,
+  compressed_toast_size bigint NOT NULL,
+  compressed_index_size bigint NOT NULL,
+  numrows_pre_compression bigint,
+  numrows_post_compression bigint,
+  -- table constraints
+  CONSTRAINT compression_chunk_size_pkey PRIMARY KEY (chunk_id),
+  CONSTRAINT compression_chunk_size_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE,
+  CONSTRAINT compression_chunk_size_compressed_chunk_id_fkey FOREIGN KEY (compressed_chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE
+);
+
+INSERT INTO _timescaledb_catalog.compression_chunk_size
+(chunk_id, compressed_chunk_id, uncompressed_heap_size, uncompressed_toast_size,
+  uncompressed_index_size, compressed_heap_size, compressed_toast_size,
+  compressed_index_size, numrows_pre_compression, numrows_post_compression)
+SELECT chunk_id, compressed_chunk_id, uncompressed_heap_size, uncompressed_toast_size,
+  uncompressed_index_size, compressed_heap_size, compressed_toast_size,
+  compressed_index_size, numrows_pre_compression, numrows_post_compression
+FROM _timescaledb_internal.compression_chunk_size_tmp;
+
+DROP TABLE _timescaledb_internal.compression_chunk_size_tmp;
+
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.compression_chunk_size', '');
+
+GRANT SELECT ON _timescaledb_catalog.compression_chunk_size TO PUBLIC;
+
+-- End modify `_timescaledb_catalog.compression_chunk_size`

src/telemetry/stats.c

@@ -335,6 +335,7 @@ add_chunk_stats(HyperStats *stats, Form_pg_class class, const Chunk *chunk,
 		stats->uncompressed_toast_size += fd_compr->uncompressed_toast_size;
 		stats->uncompressed_row_count += fd_compr->numrows_pre_compression;
 		stats->compressed_row_count += fd_compr->numrows_post_compression;
+		stats->compressed_row_frozen_count += fd_compr->numrows_frozen;
 
 		/* Also add compressed sizes to total number for entire table */
 		stats->storage.relsize.heap_size += fd_compr->compressed_heap_size;

src/telemetry/stats.h

@@ -64,6 +64,7 @@ typedef struct HyperStats
 	int64 compressed_indexes_size;
 	int64 compressed_toast_size;
 	int64 compressed_row_count;
+	int64 compressed_row_frozen_count;
 	int64 uncompressed_heap_size;
 	int64 uncompressed_indexes_size;
 	int64 uncompressed_toast_size;

src/telemetry/telemetry.c

@@ -604,6 +604,7 @@ format_iso8601(Datum value)
 #define REQ_RELKIND_COMPRESSED_TOAST_SIZE "compressed_toast_size"
 #define REQ_RELKIND_COMPRESSED_INDEXES_SIZE "compressed_indexes_size"
 #define REQ_RELKIND_COMPRESSED_ROWCOUNT "compressed_row_count"
+#define REQ_RELKIND_COMPRESSED_ROWCOUNT_FROZEN "compressed_row_count_frozen"
 
 #define REQ_RELKIND_CAGG_ON_DISTRIBUTED_HYPERTABLE_COUNT "num_caggs_on_distributed_hypertables"
 #define REQ_RELKIND_CAGG_USES_REAL_TIME_AGGREGATION_COUNT "num_caggs_using_real_time_aggregation"
@@ -639,6 +640,9 @@ add_compression_stats_object(JsonbParseState *parse_state, StatsRelType reltype,
 	ts_jsonb_add_int64(parse_state,
 					   REQ_RELKIND_COMPRESSED_INDEXES_SIZE,
 					   hs->compressed_indexes_size);
+	ts_jsonb_add_int64(parse_state,
+					   REQ_RELKIND_COMPRESSED_ROWCOUNT_FROZEN,
+					   hs->compressed_row_frozen_count);
 	ts_jsonb_add_int64(parse_state, REQ_RELKIND_UNCOMPRESSED_ROWCOUNT, hs->uncompressed_row_count);
 	ts_jsonb_add_int64(parse_state, REQ_RELKIND_UNCOMPRESSED_HEAP_SIZE, hs->uncompressed_heap_size);
 	ts_jsonb_add_int64(parse_state,

src/ts_catalog/catalog.h

@@ -1289,6 +1289,7 @@ typedef enum Anum_compression_chunk_size
 	Anum_compression_chunk_size_compressed_index_size,
 	Anum_compression_chunk_size_numrows_pre_compression,
 	Anum_compression_chunk_size_numrows_post_compression,
+	Anum_compression_chunk_size_numrows_frozen,
 	_Anum_compression_chunk_size_max,
 } Anum_compression_chunk_size;
 
@@ -1306,6 +1307,7 @@ typedef struct FormData_compression_chunk_size
 	int64 compressed_index_size;
 	int64 numrows_pre_compression;
 	int64 numrows_post_compression;
+	int64 numrows_frozen;
 } FormData_compression_chunk_size;
 
 typedef FormData_compression_chunk_size *Form_compression_chunk_size;

tsl/src/chunk_copy.c

@@ -549,6 +549,7 @@ chunk_copy_get_source_compressed_chunk_stats(ChunkCopy *cc)
 	cc->fd_ccs.compressed_index_size = atoll(PQgetvalue(res, 0, 5));
 	cc->fd_ccs.numrows_pre_compression = atoll(PQgetvalue(res, 0, 6));
 	cc->fd_ccs.numrows_post_compression = atoll(PQgetvalue(res, 0, 7));
+	cc->fd_ccs.numrows_frozen = 0;
 
 	ts_dist_cmd_close_response(dist_res);
 }

tsl/src/compression/api.c

@@ -58,7 +58,8 @@ typedef struct CompressChunkCxt
 static void
 compression_chunk_size_catalog_insert(int32 src_chunk_id, const RelationSize *src_size,
 									  int32 compress_chunk_id, const RelationSize *compress_size,
-									  int64 rowcnt_pre_compression, int64 rowcnt_post_compression)
+									  int64 rowcnt_pre_compression, int64 rowcnt_post_compression,
+									  int64 rowcnt_frozen)
 {
 	Catalog *catalog = ts_catalog_get();
 	Relation rel;
@@ -93,6 +94,8 @@ compression_chunk_size_catalog_insert(int32 src_chunk_id, const RelationSize *sr
 		Int64GetDatum(rowcnt_pre_compression);
 	values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_post_compression)] =
 		Int64GetDatum(rowcnt_post_compression);
+	values[AttrNumberGetAttrOffset(Anum_compression_chunk_size_numrows_frozen)] =
+		Int64GetDatum(rowcnt_frozen);
 
 	ts_catalog_database_info_become_owner(ts_catalog_database_info_get(), &sec_ctx);
 	ts_catalog_insert_values(rel, desc, values, nulls);
@@ -536,7 +539,8 @@ compress_chunk_impl(Oid hypertable_relid, Oid chunk_relid)
 											  compress_ht_chunk->fd.id,
 											  &after_size,
 											  cstat.rowcnt_pre_compression,
-											  cstat.rowcnt_post_compression);
+											  cstat.rowcnt_post_compression,
+											  cstat.rowcnt_frozen);
 
 		/* Copy chunk constraints (including fkey) to compressed chunk.
 		 * Do this after compressing the chunk to avoid holding strong, unnecessary locks on the
@@ -833,7 +837,8 @@ tsl_create_compressed_chunk(PG_FUNCTION_ARGS)
 										  compress_ht_chunk->fd.id,
 										  &compressed_size,
 										  numrows_pre_compression,
-										  numrows_post_compression);
+										  numrows_post_compression,
+										  0);
 
 	chunk_was_compressed = ts_chunk_is_compressed(cxt.srcht_chunk);
 	ts_chunk_set_compressed_chunk(cxt.srcht_chunk, compress_ht_chunk->fd.id);

tsl/src/compression/compression.c

@@ -451,6 +451,12 @@ compress_chunk(Oid in_table, Oid out_table, const ColumnCompressionInfo **column
 	table_close(in_rel, NoLock);
 	cstat.rowcnt_pre_compression = row_compressor.rowcnt_pre_compression;
 	cstat.rowcnt_post_compression = row_compressor.num_compressed_rows;
+
+	if ((insert_options & HEAP_INSERT_FROZEN) == HEAP_INSERT_FROZEN)
+		cstat.rowcnt_frozen = row_compressor.num_compressed_rows;
+	else
+		cstat.rowcnt_frozen = 0;
+
 	return cstat;
 }
 

tsl/src/compression/compression.h

@@ -201,6 +201,7 @@ typedef struct CompressionStats
 {
 	int64 rowcnt_pre_compression;
 	int64 rowcnt_post_compression;
+	int64 rowcnt_frozen;
 } CompressionStats;
 
 typedef struct PerColumn

tsl/test/expected/bgw_custom-14.out

@@ -972,7 +972,19 @@ SELECT count(*) = 0
 (1 row)
 
 -- cleanup
+SELECT _timescaledb_functions.stop_background_workers();
+ stop_background_workers 
+-------------------------
+ t
+(1 row)
+
 DROP TABLE sensor_data;
+SELECT _timescaledb_functions.restart_background_workers();
+ restart_background_workers 
+----------------------------
+ t
+(1 row)
+
 -- Github issue #5537
 -- Proc that waits until the given job enters the expected state
 CREATE OR REPLACE PROCEDURE wait_for_job_status(job_param_id INTEGER, expected_status TEXT, spins INTEGER=:TEST_SPINWAIT_ITERS)

tsl/test/expected/bgw_custom-15.out

@@ -972,7 +972,19 @@ SELECT count(*) = 0
 (1 row)
 
 -- cleanup
+SELECT _timescaledb_functions.stop_background_workers();
+ stop_background_workers 
+-------------------------
+ t
+(1 row)
+
 DROP TABLE sensor_data;
+SELECT _timescaledb_functions.restart_background_workers();
+ restart_background_workers 
+----------------------------
+ t
+(1 row)
+
 -- Github issue #5537
 -- Proc that waits until the given job enters the expected state
 CREATE OR REPLACE PROCEDURE wait_for_job_status(job_param_id INTEGER, expected_status TEXT, spins INTEGER=:TEST_SPINWAIT_ITERS)

tsl/test/expected/compression.out

@@ -135,6 +135,7 @@ compressed_toast_size    | 8192
 compressed_index_size    | 16384
 numrows_pre_compression  | 1
 numrows_post_compression | 1
+numrows_frozen           | 1
 -[ RECORD 2 ]------------+------
 chunk_id                 | 2
 compressed_chunk_id      | 5
@@ -146,6 +147,7 @@ compressed_toast_size    | 8192
 compressed_index_size    | 16384
 numrows_pre_compression  | 1
 numrows_post_compression | 1
+numrows_frozen           | 1
 
 \x
 select  ch1.id, ch1.schema_name, ch1.table_name ,  ch2.table_name as compress_table

tsl/test/expected/compression_bgw-14.out

@@ -72,7 +72,7 @@ from chunk_compression_stats('conditions') where compression_status like 'Compre
  _hyper_1_1_chunk | 32 kB        | 40 kB
 (1 row)
 
-SELECT * FROM _timescaledb_catalog.chunk ORDER BY id;
+SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, dropped, status, osm_chunk FROM _timescaledb_catalog.chunk ORDER BY id;
  id | hypertable_id |      schema_name      |        table_name        | compressed_chunk_id | dropped | status | osm_chunk 
 ----+---------------+-----------------------+--------------------------+---------------------+---------+--------+-----------
   1 |             1 | _timescaledb_internal | _hyper_1_1_chunk         |                   4 | f       |      1 | f
@@ -274,7 +274,6 @@ ERROR:  permission denied to start background process as role "nologin_role"
 \set ON_ERROR_STOP 1
 DROP TABLE test_table_nologin;
 RESET ROLE;
-REVOKE NOLOGIN_ROLE FROM :ROLE_DEFAULT_PERM_USER;
 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
 CREATE TABLE conditions(
     time TIMESTAMPTZ NOT NULL,

tsl/test/expected/compression_bgw-15.out

@@ -72,7 +72,7 @@ from chunk_compression_stats('conditions') where compression_status like 'Compre
  _hyper_1_1_chunk | 32 kB        | 40 kB
 (1 row)
 
-SELECT * FROM _timescaledb_catalog.chunk ORDER BY id;
+SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, dropped, status, osm_chunk FROM _timescaledb_catalog.chunk ORDER BY id;
  id | hypertable_id |      schema_name      |        table_name        | compressed_chunk_id | dropped | status | osm_chunk 
 ----+---------------+-----------------------+--------------------------+---------------------+---------+--------+-----------
   1 |             1 | _timescaledb_internal | _hyper_1_1_chunk         |                   4 | f       |      1 | f
@@ -274,7 +274,6 @@ ERROR:  permission denied to start background process as role "nologin_role"
 \set ON_ERROR_STOP 1
 DROP TABLE test_table_nologin;
 RESET ROLE;
-REVOKE NOLOGIN_ROLE FROM :ROLE_DEFAULT_PERM_USER;
 \c :TEST_DBNAME :ROLE_DEFAULT_PERM_USER
 CREATE TABLE conditions(
     time TIMESTAMPTZ NOT NULL,