Add metadata for chunk creation time

- Added creation_time attribute to timescaledb catalog table "chunk".
  Also, updated corresponding view timescaledb_information.chunks to
  include chunk_creation_time attribute.
- A newly created chunk is assigned the creation time during chunk
  creation to handle new partition range for give dimension (Time/
  SERIAL/BIGSERIAL/INT/...).
- In case of an already existing chunk, the creation time is updated as
  part of running upgrade script. The current timestamp (now()) at the
  time of upgrade has been assigned as chunk creation time.
- Similarly, downgrade script is updated to drop the attribute
  creation_time from catalog table "chunk".
- All relevant queries/views/test output has been updated accordingly.

.unreleased/feature_6062

@@ -0,0 +1 @@
+Implements: #6062 Add metadata for chunk creation time

sql/pre_install/tables.sql

@@ -190,6 +190,7 @@ CREATE TABLE _timescaledb_catalog.chunk (
   dropped boolean NOT NULL DEFAULT FALSE,
   status integer NOT NULL DEFAULT 0,
   osm_chunk boolean NOT NULL DEFAULT FALSE,
+  creation_time timestamptz NOT NULL,
   -- table constraints
   CONSTRAINT chunk_pkey PRIMARY KEY (id),
   CONSTRAINT chunk_schema_name_table_name_key UNIQUE (schema_name, table_name),
@@ -205,6 +206,7 @@ CREATE INDEX chunk_compressed_chunk_id_idx ON _timescaledb_catalog.chunk (compre
 --Another option would be to use the status field to identify a OSM chunk. However bit
 --operations only work on varbit datatype and not integer datatype.
 CREATE INDEX chunk_osm_chunk_idx ON _timescaledb_catalog.chunk (osm_chunk, hypertable_id);
+CREATE INDEX chunk_hypertable_id_creation_time_idx ON _timescaledb_catalog.chunk(hypertable_id, creation_time);
 
 SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk', '');
 SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_id_seq', '');

sql/updates/latest-dev.sql

@@ -167,3 +167,110 @@ ALTER FUNCTION _timescaledb_internal.finalize_agg_ffunc(internal,text,name,name,
 ALTER FUNCTION _timescaledb_internal.finalize_agg_sfunc(internal,text,name,name,name[],bytea,anyelement) SET SCHEMA _timescaledb_functions;
 ALTER FUNCTION _timescaledb_internal.partialize_agg(anyelement) SET SCHEMA _timescaledb_functions;
 
+-- add a new column creation_time to chunk catalog table
+ALTER TABLE _timescaledb_catalog.chunk ADD COLUMN creation_time TIMESTAMPTZ;
+
+UPDATE
+    _timescaledb_catalog.chunk c
+SET
+    creation_time = (pg_catalog.pg_stat_file(pg_catalog.pg_relation_filepath(r.oid))).change
+FROM
+    pg_class r, pg_namespace n
+WHERE
+    r.relnamespace = n.oid
+    AND r.relname = c.table_name
+    AND n.nspname = c.schema_name
+    AND r.relkind = 'r'
+    AND c.dropped IS FALSE;
+UPDATE _timescaledb_catalog.chunk SET creation_time = now() WHERE creation_time IS NULL;
+
+CREATE INDEX chunk_hypertable_id_creation_time_idx ON _timescaledb_catalog.chunk(hypertable_id, creation_time);
+
+ALTER TABLE _timescaledb_catalog.chunk
+  ALTER COLUMN creation_time SET NOT NULL;
+
+-- update chunk view
+DROP VIEW IF EXISTS timescaledb_information.chunks;
+
+CREATE OR REPLACE VIEW timescaledb_information.chunks AS
+SELECT hypertable_schema,
+  hypertable_name,
+  schema_name AS chunk_schema,
+  chunk_name,
+  primary_dimension,
+  primary_dimension_type,
+  range_start,
+  range_end,
+  integer_range_start AS range_start_integer,
+  integer_range_end AS range_end_integer,
+  is_compressed,
+  chunk_table_space AS chunk_tablespace,
+  node_list AS data_nodes,
+  creation_time AS chunk_creation_time
+FROM (
+  SELECT ht.schema_name AS hypertable_schema,
+    ht.table_name AS hypertable_name,
+    srcch.schema_name AS schema_name,
+    srcch.table_name AS chunk_name,
+    dim.column_name AS primary_dimension,
+    dim.column_type AS primary_dimension_type,
+    row_number() OVER (PARTITION BY chcons.chunk_id ORDER BY dim.id) AS chunk_dimension_num,
+    CASE WHEN (dim.column_type = 'TIMESTAMP'::regtype
+      OR dim.column_type = 'TIMESTAMPTZ'::regtype
+      OR dim.column_type = 'DATE'::regtype) THEN
+      _timescaledb_functions.to_timestamp(dimsl.range_start)
+    ELSE
+      NULL
+    END AS range_start,
+    CASE WHEN (dim.column_type = 'TIMESTAMP'::regtype
+      OR dim.column_type = 'TIMESTAMPTZ'::regtype
+      OR dim.column_type = 'DATE'::regtype) THEN
+      _timescaledb_functions.to_timestamp(dimsl.range_end)
+    ELSE
+      NULL
+    END AS range_end,
+    CASE WHEN (dim.column_type = 'TIMESTAMP'::regtype
+      OR dim.column_type = 'TIMESTAMPTZ'::regtype
+      OR dim.column_type = 'DATE'::regtype) THEN
+      NULL
+    ELSE
+      dimsl.range_start
+    END AS integer_range_start,
+    CASE WHEN (dim.column_type = 'TIMESTAMP'::regtype
+      OR dim.column_type = 'TIMESTAMPTZ'::regtype
+      OR dim.column_type = 'DATE'::regtype) THEN
+      NULL
+    ELSE
+      dimsl.range_end
+    END AS integer_range_end,
+    CASE WHEN (srcch.status & 1 = 1) THEN --distributed compress_chunk() has definitely been called
+                                          --remote chunk compression status still uncertain
+        TRUE
+    ELSE FALSE --remote chunk compression status uncertain
+    END AS is_compressed,
+    pgtab.spcname AS chunk_table_space,
+    chdn.node_list,
+	srcch.creation_time AS creation_time
+  FROM _timescaledb_catalog.chunk srcch
+    INNER JOIN _timescaledb_catalog.hypertable ht ON ht.id = srcch.hypertable_id
+    INNER JOIN _timescaledb_catalog.chunk_constraint chcons ON srcch.id = chcons.chunk_id
+    INNER JOIN _timescaledb_catalog.dimension dim ON srcch.hypertable_id = dim.hypertable_id
+    INNER JOIN _timescaledb_catalog.dimension_slice dimsl ON dim.id = dimsl.dimension_id
+      AND chcons.dimension_slice_id = dimsl.id
+    INNER JOIN (
+      SELECT relname,
+        reltablespace,
+        nspname AS schema_name
+      FROM pg_class,
+        pg_namespace
+      WHERE pg_class.relnamespace = pg_namespace.oid) cl ON srcch.table_name = cl.relname
+      AND srcch.schema_name = cl.schema_name
+    LEFT OUTER JOIN pg_tablespace pgtab ON pgtab.oid = reltablespace
+  LEFT OUTER JOIN (
+    SELECT chunk_id,
+      array_agg(node_name ORDER BY node_name) AS node_list
+    FROM _timescaledb_catalog.chunk_data_node
+    GROUP BY chunk_id) chdn ON srcch.id = chdn.chunk_id
+  WHERE srcch.dropped IS FALSE AND srcch.osm_chunk IS FALSE
+    AND ht.compression_state != 2 ) finalq
+WHERE chunk_dimension_num = 1;

sql/updates/reverse-dev.sql

@@ -273,3 +273,120 @@ ALTER FUNCTION _timescaledb_functions.finalize_agg_sfunc(internal,text,name,name
 ALTER FUNCTION _timescaledb_functions.partialize_agg(anyelement) SET SCHEMA _timescaledb_internal;
 ALTER AGGREGATE _timescaledb_functions.finalize_agg(text,name,name,name[][],bytea,anyelement) SET SCHEMA _timescaledb_internal;
 
+--
+-- Rebuild the catalog table `_timescaledb_catalog.chunk`
+--
+-- We need to recreate the catalog from scratch because when we drop a column
+-- Postgres marks `pg_attribute.attisdropped=TRUE` instead of removing it from
+-- the `pg_catalog.pg_attribute` table.
+--
+-- If we downgrade and upgrade the extension without rebuilding the catalog table it
+-- will mess up `pg_attribute.attnum` and we will end up with issues when trying
+-- to update data in those catalog tables.
+
+-- Recreate _timescaledb_catalog.chunk table --
+CREATE TABLE _timescaledb_internal.chunk_tmp
+AS SELECT * from _timescaledb_catalog.chunk;
+
+CREATE TABLE _timescaledb_internal.tmp_chunk_seq_value AS
+SELECT last_value, is_called FROM _timescaledb_catalog.chunk_id_seq;
+
+--drop foreign keys on chunk table
+ALTER TABLE _timescaledb_catalog.chunk_constraint DROP CONSTRAINT
+chunk_constraint_chunk_id_fkey;
+ALTER TABLE _timescaledb_catalog.chunk_index DROP CONSTRAINT
+chunk_index_chunk_id_fkey;
+ALTER TABLE _timescaledb_catalog.chunk_data_node DROP CONSTRAINT
+chunk_data_node_chunk_id_fkey;
+ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats DROP CONSTRAINT
+bgw_policy_chunk_stats_chunk_id_fkey;
+ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT
+compression_chunk_size_chunk_id_fkey;
+ALTER TABLE _timescaledb_catalog.compression_chunk_size DROP CONSTRAINT
+compression_chunk_size_compressed_chunk_id_fkey;
+ALTER TABLE _timescaledb_catalog.chunk_copy_operation DROP CONSTRAINT
+chunk_copy_operation_chunk_id_fkey;
+
+--drop dependent views
+DROP VIEW IF EXISTS timescaledb_information.hypertables;
+DROP VIEW IF EXISTS timescaledb_information.chunks;
+DROP VIEW IF EXISTS _timescaledb_internal.hypertable_chunk_local_size;
+DROP VIEW IF EXISTS _timescaledb_internal.compressed_chunk_stats;
+DROP VIEW IF EXISTS timescaledb_experimental.chunk_replication_status;
+
+ALTER EXTENSION timescaledb DROP TABLE _timescaledb_catalog.chunk;
+ALTER EXTENSION timescaledb DROP SEQUENCE _timescaledb_catalog.chunk_id_seq;
+DROP TABLE _timescaledb_catalog.chunk;
+
+CREATE SEQUENCE _timescaledb_catalog.chunk_id_seq MINVALUE 1;
+
+-- now create table without self referential foreign key
+CREATE TABLE _timescaledb_catalog.chunk (
+  id integer NOT NULL DEFAULT nextval('_timescaledb_catalog.chunk_id_seq'),
+  hypertable_id int NOT NULL,
+  schema_name name NOT NULL,
+  table_name name NOT NULL,
+  compressed_chunk_id integer ,
+  dropped boolean NOT NULL DEFAULT FALSE,
+  status integer NOT NULL DEFAULT 0,
+  osm_chunk boolean NOT NULL DEFAULT FALSE,
+  -- table constraints
+  CONSTRAINT chunk_pkey PRIMARY KEY (id),
+  CONSTRAINT chunk_schema_name_table_name_key UNIQUE (schema_name, table_name)
+);
+
+INSERT INTO _timescaledb_catalog.chunk
+( id, hypertable_id, schema_name, table_name,
+  compressed_chunk_id, dropped, status, osm_chunk)
+SELECT id, hypertable_id, schema_name, table_name,
+  compressed_chunk_id, dropped, status, osm_chunk
+FROM _timescaledb_internal.chunk_tmp;
+
+--add indexes to the chunk table
+CREATE INDEX chunk_hypertable_id_idx ON _timescaledb_catalog.chunk (hypertable_id);
+CREATE INDEX chunk_compressed_chunk_id_idx ON _timescaledb_catalog.chunk (compressed_chunk_id);
+CREATE INDEX chunk_osm_chunk_idx ON _timescaledb_catalog.chunk (osm_chunk, hypertable_id);
+
+ALTER SEQUENCE _timescaledb_catalog.chunk_id_seq OWNED BY _timescaledb_catalog.chunk.id;
+SELECT setval('_timescaledb_catalog.chunk_id_seq', last_value, is_called) FROM _timescaledb_internal.tmp_chunk_seq_value;
+
+-- add self referential foreign key
+ALTER TABLE _timescaledb_catalog.chunk ADD CONSTRAINT chunk_compressed_chunk_id_fkey FOREIGN KEY ( compressed_chunk_id )
+ REFERENCES _timescaledb_catalog.chunk( id );
+
+--add foreign key constraint
+ALTER TABLE _timescaledb_catalog.chunk
+      ADD CONSTRAINT chunk_hypertable_id_fkey
+      FOREIGN KEY (hypertable_id) REFERENCES _timescaledb_catalog.hypertable (id);
+
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk', '');
+SELECT pg_catalog.pg_extension_config_dump('_timescaledb_catalog.chunk_id_seq', '');
+
+--add the foreign key constraints
+ALTER TABLE _timescaledb_catalog.chunk_constraint ADD CONSTRAINT
+chunk_constraint_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id);
+ALTER TABLE _timescaledb_catalog.chunk_index ADD CONSTRAINT
+chunk_index_chunk_id_fkey FOREIGN KEY (chunk_id)
+REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.chunk_data_node ADD CONSTRAINT
+chunk_data_node_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk(id);
+ALTER TABLE _timescaledb_internal.bgw_policy_chunk_stats ADD CONSTRAINT
+bgw_policy_chunk_stats_chunk_id_fkey FOREIGN KEY (chunk_id)
+REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT
+compression_chunk_size_chunk_id_fkey FOREIGN KEY (chunk_id)
+REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.compression_chunk_size ADD CONSTRAINT
+compression_chunk_size_compressed_chunk_id_fkey FOREIGN KEY (compressed_chunk_id)
+REFERENCES _timescaledb_catalog.chunk(id) ON DELETE CASCADE;
+ALTER TABLE _timescaledb_catalog.chunk_copy_operation ADD CONSTRAINT
+chunk_copy_operation_chunk_id_fkey FOREIGN KEY (chunk_id) REFERENCES _timescaledb_catalog.chunk (id) ON DELETE CASCADE;
+
+--cleanup
+DROP TABLE _timescaledb_internal.chunk_tmp;
+DROP TABLE _timescaledb_internal.tmp_chunk_seq_value;
+
+GRANT SELECT ON _timescaledb_catalog.chunk_id_seq TO PUBLIC;
+GRANT SELECT ON _timescaledb_catalog.chunk TO PUBLIC;
+
+-- end recreate _timescaledb_catalog.chunk table --

sql/views.sql

@@ -176,7 +176,8 @@ SELECT hypertable_schema,
   integer_range_end AS range_end_integer,
   is_compressed,
   chunk_table_space AS chunk_tablespace,
-  node_list AS data_nodes
+  node_list AS data_nodes,
+  creation_time AS chunk_creation_time
 FROM (
   SELECT ht.schema_name AS hypertable_schema,
     ht.table_name AS hypertable_name,
@@ -219,7 +220,8 @@ FROM (
     ELSE FALSE --remote chunk compression status uncertain
     END AS is_compressed,
     pgtab.spcname AS chunk_table_space,
-    chdn.node_list
+    chdn.node_list,
+	srcch.creation_time AS creation_time
   FROM _timescaledb_catalog.chunk srcch
     INNER JOIN _timescaledb_catalog.hypertable ht ON ht.id = srcch.hypertable_id
     INNER JOIN _timescaledb_catalog.chunk_constraint chcons ON srcch.id = chcons.chunk_id

src/chunk.c

@@ -163,6 +163,7 @@ chunk_formdata_make_tuple(const FormData_chunk *fd, TupleDesc desc)
 	values[AttrNumberGetAttrOffset(Anum_chunk_dropped)] = BoolGetDatum(fd->dropped);
 	values[AttrNumberGetAttrOffset(Anum_chunk_status)] = Int32GetDatum(fd->status);
 	values[AttrNumberGetAttrOffset(Anum_chunk_osm_chunk)] = BoolGetDatum(fd->osm_chunk);
+	values[AttrNumberGetAttrOffset(Anum_chunk_creation_time)] = Int64GetDatum(fd->creation_time);
 
 	return heap_form_tuple(desc, values, nulls);
 }
@@ -185,6 +186,7 @@ ts_chunk_formdata_fill(FormData_chunk *fd, const TupleInfo *ti)
 	Assert(!nulls[AttrNumberGetAttrOffset(Anum_chunk_dropped)]);
 	Assert(!nulls[AttrNumberGetAttrOffset(Anum_chunk_status)]);
 	Assert(!nulls[AttrNumberGetAttrOffset(Anum_chunk_osm_chunk)]);
+	Assert(!nulls[AttrNumberGetAttrOffset(Anum_chunk_creation_time)]);
 
 	fd->id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_id)]);
 	fd->hypertable_id = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_hypertable_id)]);
@@ -204,6 +206,7 @@ ts_chunk_formdata_fill(FormData_chunk *fd, const TupleInfo *ti)
 	fd->dropped = DatumGetBool(values[AttrNumberGetAttrOffset(Anum_chunk_dropped)]);
 	fd->status = DatumGetInt32(values[AttrNumberGetAttrOffset(Anum_chunk_status)]);
 	fd->osm_chunk = DatumGetBool(values[AttrNumberGetAttrOffset(Anum_chunk_osm_chunk)]);
+	fd->creation_time = DatumGetInt64(values[AttrNumberGetAttrOffset(Anum_chunk_creation_time)]);
 
 	if (should_free)
 		heap_freetuple(tuple);
@@ -973,6 +976,7 @@ chunk_create_object(const Hypertable *ht, Hypercube *cube, const char *schema_na
 	chunk->cube = cube;
 	chunk->hypertable_relid = ht->main_table_relid;
 	namestrcpy(&chunk->fd.schema_name, schema_name);
+	chunk->fd.creation_time = GetCurrentTimestamp();
 
 	if (NULL == table_name || table_name[0] == '\0')
 	{

src/ts_catalog/catalog.c

@@ -178,6 +178,7 @@ static const TableIndexDef catalog_table_index_definitions[_MAX_CATALOG_TABLES]
 			[CHUNK_SCHEMA_NAME_INDEX] = "chunk_schema_name_table_name_key",
 			[CHUNK_COMPRESSED_CHUNK_ID_INDEX] = "chunk_compressed_chunk_id_idx",
 			[CHUNK_OSM_CHUNK_INDEX] = "chunk_osm_chunk_idx",
+			[CHUNK_HYPERTABLE_ID_CREATION_TIME_INDEX] = "chunk_hypertable_id_creation_time_idx",
 		},
 	},
 	[CHUNK_CONSTRAINT] = {

src/ts_catalog/catalog.h

@@ -413,6 +413,7 @@ enum Anum_chunk
 	Anum_chunk_dropped,
 	Anum_chunk_status,
 	Anum_chunk_osm_chunk,
+	Anum_chunk_creation_time,
 	_Anum_chunk_max,
 };
 
@@ -428,6 +429,7 @@ typedef struct FormData_chunk
 	bool dropped;
 	int32 status;
 	bool osm_chunk;
+	TimestampTz creation_time;
 } FormData_chunk;
 
 typedef FormData_chunk *Form_chunk;
@@ -439,6 +441,7 @@ enum
 	CHUNK_SCHEMA_NAME_INDEX,
 	CHUNK_COMPRESSED_CHUNK_ID_INDEX,
 	CHUNK_OSM_CHUNK_INDEX,
+	CHUNK_HYPERTABLE_ID_CREATION_TIME_INDEX,
 	_MAX_CHUNK_INDEX,
 };
 
@@ -469,6 +472,12 @@ enum Anum_chunk_osm_chunk_idx
 	Anum_chunk_osm_chunk_idx_hypertable_id,
 };
 
+enum Anum_chunk_hypertable_id_creation_time_idx
+{
+	Anum_chunk_hypertable_id_creation_time_idx_hypertable_id = 1,
+	Anum_chunk_hypertable_id_creation_time_idx_creation_time,
+};
+
 /************************************
  *
  * Chunk constraint table definitions
@@ -1316,7 +1325,7 @@ typedef enum Anum_compression_chunk_size_pkey
  * The maximum number of indexes a catalog table can have.
  * This needs to be bumped in case of new catalog tables that have more indexes.
  */
-#define _MAX_TABLE_INDEXES 5
+#define _MAX_TABLE_INDEXES 6
 /************************************
  *
  * Remote txn table of 2pc commits

test/expected/alter.out

@@ -186,23 +186,23 @@ SELECT relname, reloptions FROM pg_class WHERE relname IN ('_hyper_2_3_chunk','_
 
 -- Need superuser to ALTER chunks in _timescaledb_internal schema
 \c :TEST_DBNAME :ROLE_SUPERUSER
-SELECT * FROM _timescaledb_catalog.chunk WHERE id = 2;
+SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, dropped, status, osm_chunk FROM _timescaledb_catalog.chunk WHERE id = 2;
  id | hypertable_id |      schema_name      |    table_name    | compressed_chunk_id | dropped | status | osm_chunk 
 ----+---------------+-----------------------+------------------+---------------------+---------+--------+-----------
   2 |             2 | _timescaledb_internal | _hyper_2_2_chunk |                     | f       |      0 | f
 (1 row)
 
 -- Rename chunk
 ALTER TABLE _timescaledb_internal._hyper_2_2_chunk RENAME TO new_chunk_name;
-SELECT * FROM _timescaledb_catalog.chunk WHERE id = 2;
+SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, dropped, status, osm_chunk FROM _timescaledb_catalog.chunk WHERE id = 2;
  id | hypertable_id |      schema_name      |   table_name   | compressed_chunk_id | dropped | status | osm_chunk 
 ----+---------------+-----------------------+----------------+---------------------+---------+--------+-----------
   2 |             2 | _timescaledb_internal | new_chunk_name |                     | f       |      0 | f
 (1 row)
 
 -- Set schema
 ALTER TABLE _timescaledb_internal.new_chunk_name SET SCHEMA public;
-SELECT * FROM _timescaledb_catalog.chunk WHERE id = 2;
+SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, dropped, status, osm_chunk FROM _timescaledb_catalog.chunk WHERE id = 2;
  id | hypertable_id | schema_name |   table_name   | compressed_chunk_id | dropped | status | osm_chunk 
 ----+---------------+-------------+----------------+---------------------+---------+--------+-----------
   2 |             2 | public      | new_chunk_name |                     | f       |      0 | f
@@ -680,7 +680,7 @@ SELECT * from _timescaledb_catalog.hypertable;
  12 | public      | my_table   | new_associated_schema  | _hyper_12               |              1 | _timescaledb_functions   | calculate_chunk_interval |                 0 |                 0 |                          |                   
 (1 row)
 
-SELECT * from _timescaledb_catalog.chunk;
+SELECT id, hypertable_id, schema_name, table_name, compressed_chunk_id, dropped, status, osm_chunk from _timescaledb_catalog.chunk;
  id | hypertable_id |      schema_name      |     table_name     | compressed_chunk_id | dropped | status | osm_chunk 
 ----+---------------+-----------------------+--------------------+---------------------+---------+--------+-----------
  24 |            12 | new_associated_schema | _hyper_12_24_chunk |                     | f       |      0 | f