Add metadata for chunk creation time

- Added creation_time attribute to timescaledb catalog table "chunk".
  Also, updated corresponding view timescaledb_information.chunks to
  include chunk_creation_time attribute.
- A newly created chunk is assigned the creation time during chunk
  creation to handle new partition range for give dimension (Time/
  SERIAL/BIGSERIAL/INT/...).
- In case of an already existing chunk, the creation time is updated as
  part of running upgrade script. The current timestamp (now()) at the
  time of upgrade has been assigned as chunk creation time.
- Similarly, downgrade script is updated to drop the attribute
  creation_time from catalog table "chunk".
- All relevant queries/views/test output have been updated accordingly.

Co-authored-by: Nikhil Sontakke <[email protected]>

.unreleased/feature_6062

@@ -0,0 +1 @@
+Implements: #6062 Add metadata for chunk creation time

sql/pre_install/tables.sql

@@ -191,6 +191,7 @@ CREATE TABLE _timescaledb_catalog.chunk (
   dropped boolean NOT NULL DEFAULT FALSE,
   status integer NOT NULL DEFAULT 0,
   osm_chunk boolean NOT NULL DEFAULT FALSE,
+  creation_time timestamptz NOT NULL,
   -- table constraints
   CONSTRAINT chunk_pkey PRIMARY KEY (id),
   CONSTRAINT chunk_schema_name_table_name_key UNIQUE (schema_name, table_name),
@@ -206,6 +207,7 @@ CREATE INDEX chunk_compressed_chunk_id_idx ON _timescaledb_catalog.chunk (compre
 --Another option would be to use the status field to identify a OSM chunk. However bit
 --operations only work on varbit datatype and not integer datatype.
 CREATE INDEX chunk_osm_chunk_idx ON _timescaledb_catalog.chunk (osm_chunk, hypertable_id);
+CREATE INDEX chunk_hypertable_id_creation_time_idx ON _timescaledb_catalog.chunk(hypertable_id, creation_time);
 
 SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk', '');
 SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_id_seq', '');

sql/updates/latest-dev.sql

@@ -0,0 +1,131 @@
+--
+-- Rebuild the catalog table `_timescaledb_catalog.chunk` to
+-- add new column `creation_time`
+--
+CREATE TABLE _timescaledb_internal.chunk_tmp
+AS SELECT * from _timescaledb_catalog.chunk;
+
+CREATE TABLE _timescaledb_internal.tmp_chunk_seq_value AS
+SELECT last_value, is_called FROM _timescaledb_catalog.chunk_id_seq;
+
+--drop foreign keys on chunk table
+ALTER TABLE _timescaledb_catalog.chunk_constraint DROP CONSTRAINT
+chunk_constraint_chunk_id_fkey;
+ALTER TABLE _timescaledb_catalog.chunk_index DROP CONSTRAINT
+chunk_index_chunk_id_fkey;
+ALTER TABLE _timescaledb_catalog.chunk_data_node DROP CONSTRAINT
+chunk_data_node_chunk_id_fkey;
+ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats DROP CONSTRAINT
+bgw_policy_chunk_stats_chunk_id_fkey;
+ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT
+compression_chunk_size_chunk_id_fkey;
+ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT
+compression_chunk_size_compressed_chunk_id_fkey;
+ALTER TABLE _timescaledb_catalog.chunk_copy_operation DROP CONSTRAINT
+chunk_copy_operation_chunk_id_fkey;
+
+--drop dependent views
+DROP VIEW IF EXISTS timescaledb_information.hypertables;
+DROP VIEW IF EXISTS timescaledb_information.chunks;
+DROP VIEW IF EXISTS _timescaledb_internal.hypertable_chunk_local_size;
+DROP VIEW IF EXISTS _timescaledb_internal.compressed_chunk_stats;
+DROP VIEW IF EXISTS timescaledb_experimental.chunk_replication_status;
+
+ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.chunk;
+ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.chunk_id_seq;
+DROP TABLE _timescaledb_catalog.chunk;
+
+CREATE SEQUENCE _timescaledb_catalog.chunk_id_seq MINVALUE 1;
+
+-- now create table without self referential foreign key
+CREATE TABLE _timescaledb_catalog.chunk (
+  id integer NOT NULL DEFAULT nextval('_timescaledb_catalog.chunk_id_seq'),
+  hypertable_id int NOT NULL,
+  schema_name name NOT NULL,
+  table_name name NOT NULL,
+  compressed_chunk_id integer ,
+  dropped boolean NOT NULL DEFAULT FALSE,
+  status integer NOT NULL DEFAULT 0,
+  osm_chunk boolean NOT NULL DEFAULT FALSE,
+  creation_time timestamptz,
+  -- table constraints
+  CONSTRAINT chunk_pkey PRIMARY KEY (id),
+  CONSTRAINT chunk_schema_name_table_name_key UNIQUE (schema_name, table_name)
+);
+
+INSERT INTO _timescaledb_catalog.chunk
+( id, hypertable_id, schema_name, table_name,
+  compressed_chunk_id, dropped, status, osm_chunk)
+SELECT id, hypertable_id, schema_name, table_name,
+  compressed_chunk_id, dropped, status, osm_chunk
+FROM _timescaledb_internal.chunk_tmp;
+
+-- update creation_time for chunks
+UPDATE
+    _timescaledb_catalog.chunk c
+SET
+    creation_time = (pg_catalog.pg_stat_file(pg_catalog.pg_relation_filepath(r.oid))).modification
+FROM
+    pg_class r, pg_namespace n
+WHERE
+    r.relnamespace = n.oid
+    AND r.relname = c.table_name
+    AND n.nspname = c.schema_name
+    AND r.relkind = 'r'
+    AND c.dropped IS FALSE;
+
+-- Make sure that there are no record with empty creation time
+UPDATE _timescaledb_catalog.chunk SET creation_time = now() WHERE creation_time IS NULL;
+
+--add indexes to the chunk table
+CREATE INDEX chunk_hypertable_id_idx ON _timescaledb_catalog.chunk (hypertable_id);
+CREATE INDEX chunk_compressed_chunk_id_idx ON _timescaledb_catalog.chunk (compressed_chunk_id);
+CREATE INDEX chunk_osm_chunk_idx ON _timescaledb_catalog.chunk (osm_chunk, hypertable_id);
+CREATE INDEX chunk_hypertable_id_creation_time_idx ON _timescaledb_catalog.chunk(hypertable_id, creation_time);
+
+ALTER SEQUENCE _timescaledb_catalog.chunk_id_seq OWNED BY _timescaledb_catalog.chunk.id;
+SELECT setval('_timescaledb_catalog.chunk_id_seq', last_value, is_called) FROM _timescaledb_internal.tmp_chunk_seq_value;
+
+-- add self referential foreign key
+ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_compressed_chunk_id_fkey FOREIGN KEY ( compressed_chunk_id )
+ REFERENCES _timescaledb_catalog.chunk( id );
+
+--add foreign key constraint
+ALTER TABLE _timescaledb_catalog.chunk
+      ADD CONSTRAINT chunk_hypertable_id_fkey
+      FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id);
+
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk', '');
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_id_seq', '');
+
+-- Add non-null constraint
+ALTER TABLE _timescaledb_catalog.chunk
+  ALTER COLUMN creation_time SET NOT NULL;
+
+--add the foreign key constraints
+ALTER TABLE _timescaledb_catalog.chunk_constraint ADD CONSTRAINT
+chunk_constraint_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id);
+ALTER TABLE _timescaledb_catalog.chunk_index ADD CONSTRAINT
+chunk_index_chunk_id_fkey FOREIGN KEY (chunk_id)
+REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.chunk_data_node ADD CONSTRAINT
+chunk_data_node_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id);
+ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats ADD CONSTRAINT
+bgw_policy_chunk_stats_chunk_id_fkey FOREIGN KEY (chunk_id)
+REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT
+compression_chunk_size_chunk_id_fkey FOREIGN KEY (chunk_id)
+REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT
+compression_chunk_size_compressed_chunk_id_fkey FOREIGN KEY (compressed_chunk_id)
+REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.chunk_copy_operation ADD CONSTRAINT
+chunk_copy_operation_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE;
+
+--cleanup
+DROP TABLE _timescaledb_internal.chunk_tmp;
+DROP TABLE _timescaledb_internal.tmp_chunk_seq_value;
+
+GRANT SELECT ON _timescaledb_catalog.chunk_id_seq TO PUBLIC;
+GRANT SELECT ON _timescaledb_catalog.chunk TO PUBLIC;
+-- end recreate _timescaledb_catalog.chunk table --

sql/updates/reverse-dev.sql

@@ -0,0 +1,117 @@
+--
+-- Rebuild the catalog table `_timescaledb_catalog.chunk`
+--
+-- We need to recreate the catalog from scratch because when we drop a column
+-- Postgres marks `pg_attribute.attisdropped=TRUE` instead of removing it from
+-- the `pg_catalog.pg_attribute` table.
+--
+-- If we downgrade and upgrade the extension without rebuilding the catalog table it
+-- will mess up `pg_attribute.attnum` and we will end up with issues when trying
+-- to update data in those catalog tables.
+
+-- Recreate _timescaledb_catalog.chunk table --
+CREATE TABLE _timescaledb_internal.chunk_tmp
+AS SELECT * from _timescaledb_catalog.chunk;
+
+CREATE TABLE _timescaledb_internal.tmp_chunk_seq_value AS
+SELECT last_value, is_called FROM _timescaledb_catalog.chunk_id_seq;
+
+--drop foreign keys on chunk table
+ALTER TABLE _timescaledb_catalog.chunk_constraint DROP CONSTRAINT
+chunk_constraint_chunk_id_fkey;
+ALTER TABLE _timescaledb_catalog.chunk_index DROP CONSTRAINT
+chunk_index_chunk_id_fkey;
+ALTER TABLE _timescaledb_catalog.chunk_data_node DROP CONSTRAINT
+chunk_data_node_chunk_id_fkey;
+ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats DROP CONSTRAINT
+bgw_policy_chunk_stats_chunk_id_fkey;
+ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT
+compression_chunk_size_chunk_id_fkey;
+ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT
+compression_chunk_size_compressed_chunk_id_fkey;
+ALTER TABLE _timescaledb_catalog.chunk_copy_operation DROP CONSTRAINT
+chunk_copy_operation_chunk_id_fkey;
+
+--drop dependent views
+DROP VIEW IF EXISTS timescaledb_information.hypertables;
+DROP VIEW IF EXISTS timescaledb_information.chunks;
+DROP VIEW IF EXISTS _timescaledb_internal.hypertable_chunk_local_size;
+DROP VIEW IF EXISTS _timescaledb_internal.compressed_chunk_stats;
+DROP VIEW IF EXISTS timescaledb_experimental.chunk_replication_status;
+
+ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.chunk;
+ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.chunk_id_seq;
+DROP TABLE _timescaledb_catalog.chunk;
+
+CREATE SEQUENCE _timescaledb_catalog.chunk_id_seq MINVALUE 1;
+
+-- now create table without self referential foreign key
+CREATE TABLE _timescaledb_catalog.chunk (
+  id integer NOT NULL DEFAULT nextval('_timescaledb_catalog.chunk_id_seq'),
+  hypertable_id int NOT NULL,
+  schema_name name NOT NULL,
+  table_name name NOT NULL,
+  compressed_chunk_id integer ,
+  dropped boolean NOT NULL DEFAULT FALSE,
+  status integer NOT NULL DEFAULT 0,
+  osm_chunk boolean NOT NULL DEFAULT FALSE,
+  -- table constraints
+  CONSTRAINT chunk_pkey PRIMARY KEY (id),
+  CONSTRAINT chunk_schema_name_table_name_key UNIQUE (schema_name, table_name)
+);
+
+INSERT INTO _timescaledb_catalog.chunk
+( id, hypertable_id, schema_name, table_name,
+  compressed_chunk_id, dropped, status, osm_chunk)
+SELECT id, hypertable_id, schema_name, table_name,
+  compressed_chunk_id, dropped, status, osm_chunk
+FROM _timescaledb_internal.chunk_tmp;
+
+--add indexes to the chunk table
+CREATE INDEX chunk_hypertable_id_idx ON _timescaledb_catalog.chunk (hypertable_id);
+CREATE INDEX chunk_compressed_chunk_id_idx ON _timescaledb_catalog.chunk (compressed_chunk_id);
+CREATE INDEX chunk_osm_chunk_idx ON _timescaledb_catalog.chunk (osm_chunk, hypertable_id);
+
+ALTER SEQUENCE _timescaledb_catalog.chunk_id_seq OWNED BY _timescaledb_catalog.chunk.id;
+SELECT setval('_timescaledb_catalog.chunk_id_seq', last_value, is_called) FROM _timescaledb_internal.tmp_chunk_seq_value;
+
+-- add self referential foreign key
+ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_compressed_chunk_id_fkey FOREIGN KEY ( compressed_chunk_id )
+ REFERENCES _timescaledb_catalog.chunk( id );
+
+--add foreign key constraint
+ALTER TABLE _timescaledb_catalog.chunk
+      ADD CONSTRAINT chunk_hypertable_id_fkey
+      FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id);
+
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk', '');
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_id_seq', '');
+
+--add the foreign key constraints
+ALTER TABLE _timescaledb_catalog.chunk_constraint ADD CONSTRAINT
+chunk_constraint_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id);
+ALTER TABLE _timescaledb_catalog.chunk_index ADD CONSTRAINT
+chunk_index_chunk_id_fkey FOREIGN KEY (chunk_id)
+REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.chunk_data_node ADD CONSTRAINT
+chunk_data_node_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id);
+ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats ADD CONSTRAINT
+bgw_policy_chunk_stats_chunk_id_fkey FOREIGN KEY (chunk_id)
+REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT
+compression_chunk_size_chunk_id_fkey FOREIGN KEY (chunk_id)
+REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT
+compression_chunk_size_compressed_chunk_id_fkey FOREIGN KEY (compressed_chunk_id)
+REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.chunk_copy_operation ADD CONSTRAINT
+chunk_copy_operation_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE;
+
+--cleanup
+DROP TABLE _timescaledb_internal.chunk_tmp;
+DROP TABLE _timescaledb_internal.tmp_chunk_seq_value;
+
+GRANT SELECT ON _timescaledb_catalog.chunk_id_seq TO PUBLIC;
+GRANT SELECT ON _timescaledb_catalog.chunk TO PUBLIC;
+
+-- end recreate _timescaledb_catalog.chunk table --

sql/views.sql

@@ -176,7 +176,8 @@ SELECT hypertable_schema,
   integer_range_end AS range_end_integer,
   is_compressed,
   chunk_table_space AS chunk_tablespace,
-  node_list AS data_nodes
+  node_list AS data_nodes,
+  creation_time AS chunk_creation_time
 FROM (
   SELECT ht.schema_name AS hypertable_schema,
     ht.table_name AS hypertable_name,
@@ -219,7 +220,8 @@ FROM (
     ELSE FALSE --remote chunk compression status uncertain
     END AS is_compressed,
     pgtab.spcname AS chunk_table_space,
-    chdn.node_list
+    chdn.node_list,
+	srcch.creation_time AS creation_time
   FROM _timescaledb_catalog.chunk srcch
     INNER JOIN _timescaledb_catalog.hypertable ht ON ht.id = srcch.hypertable_id
     INNER JOIN _timescaledb_catalog.chunk_constraint chcons ON srcch.id = chcons.chunk_id

src/chunk.c

@@ -163,6 +163,7 @@ chunk_formdata_make_tuple(const FormData_chunk *fd, TupleDesc desc)
 	values[AttrNumberGetAttrOffset(Anum_chunk_dropped)] = BoolGetDatum(fd->dropped);
 	values[AttrNumberGetAttrOffset(Anum_chunk_status)] = Int32GetDatum(fd->status);
 	values[AttrNumberGetAttrOffset(Anum_chunk_osm_chunk)] = BoolGetDatum(fd->osm_chunk);
+	values[AttrNumberGetAttrOffset(Anum_chunk_creation_time)] = Int64GetDatum(fd->creation_time);
 
 	return heap_form_tuple(desc, values, nulls);
 }
@@ -185,6 +186,7 @@ ts_chunk_formdata_fill(FormData_chunk *fd, const TupleInfo *ti)
 	Assert(!nulls[AttrNumberGetAttrOffset(Anum_chunk_dropped)]);
 	Assert(!nulls[AttrNumberGetAttrOffset(Anum_chunk_status)]);
 	Assert(!nulls[AttrNumberGetAttrOffset(Anum_chunk_osm_chunk)]);
+	Assert(!nulls[AttrNumberGetAttrOffset(Anum_chunk_creation_time)]);
 
 	fd->id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_id)]);
 	fd->hypertable_id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_hypertable_id)]);
@@ -204,6 +206,7 @@ ts_chunk_formdata_fill(FormData_chunk *fd, const TupleInfo *ti)
 	fd->dropped = DatumGetBool(values[AttrNumberGetAttrOffset(Anum_chunk_dropped)]);
 	fd->status = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_status)]);
 	fd->osm_chunk = DatumGetBool(values[AttrNumberGetAttrOffset(Anum_chunk_osm_chunk)]);
+	fd->creation_time = DatumGetInt64(values[AttrNumberGetAttrOffset(Anum_chunk_creation_time)]);
 
 	if (should_free)
 		heap_freetuple(tuple);
@@ -973,6 +976,7 @@ chunk_create_object(const Hypertable *ht, Hypercube *cube, const char *schema_na
 	chunk->cube = cube;
 	chunk->hypertable_relid = ht->main_table_relid;
 	namestrcpy(&chunk->fd.schema_name, schema_name);
+	chunk->fd.creation_time = GetCurrentTimestamp();
 
 	if (NULL == table_name || table_name[0] == '\0')
 	{

src/ts_catalog/catalog.c

@@ -178,6 +178,7 @@ static const TableIndexDef catalog_table_index_definitions[_MAX_CATALOG_TABLES]
 			[CHUNK_SCHEMA_NAME_INDEX] = "chunk_schema_name_table_name_key",
 			[CHUNK_COMPRESSED_CHUNK_ID_INDEX] = "chunk_compressed_chunk_id_idx",
 			[CHUNK_OSM_CHUNK_INDEX] = "chunk_osm_chunk_idx",
+			[CHUNK_HYPERTABLE_ID_CREATION_TIME_INDEX] = "chunk_hypertable_id_creation_time_idx",
 		},
 	},
 	[CHUNK_CONSTRAINT] = {

src/ts_catalog/catalog.h

@@ -415,6 +415,7 @@ enum Anum_chunk
 	Anum_chunk_dropped,
 	Anum_chunk_status,
 	Anum_chunk_osm_chunk,
+	Anum_chunk_creation_time,
 	_Anum_chunk_max,
 };
 
@@ -430,6 +431,7 @@ typedef struct FormData_chunk
 	bool dropped;
 	int32 status;
 	bool osm_chunk;
+	TimestampTz creation_time;
 } FormData_chunk;
 
 typedef FormData_chunk *Form_chunk;
@@ -441,6 +443,7 @@ enum
 	CHUNK_SCHEMA_NAME_INDEX,
 	CHUNK_COMPRESSED_CHUNK_ID_INDEX,
 	CHUNK_OSM_CHUNK_INDEX,
+	CHUNK_HYPERTABLE_ID_CREATION_TIME_INDEX,
 	_MAX_CHUNK_INDEX,
 };
 
@@ -471,6 +474,12 @@ enum Anum_chunk_osm_chunk_idx
 	Anum_chunk_osm_chunk_idx_hypertable_id,
 };
 
+enum Anum_chunk_hypertable_id_creation_time_idx
+{
+	Anum_chunk_hypertable_id_creation_time_idx_hypertable_id = 1,
+	Anum_chunk_hypertable_id_creation_time_idx_creation_time,
+};
+
 /************************************
  *
  * Chunk constraint table definitions
@@ -1318,7 +1327,7 @@ typedef enum Anum_compression_chunk_size_pkey
  * The maximum number of indexes a catalog table can have.
  * This needs to be bumped in case of new catalog tables that have more indexes.
  */
-#define _MAX_TABLE_INDEXES 5
+#define _MAX_TABLE_INDEXES 6
 /************************************
  *
  * Remote txn table of 2pc commits