diff --git a/postgres/datadog_checks/postgres/relationsmanager.py b/postgres/datadog_checks/postgres/relationsmanager.py index 14ca6642edd97..4cd63d67084e6 100644 --- a/postgres/datadog_checks/postgres/relationsmanager.py +++ b/postgres/datadog_checks/postgres/relationsmanager.py @@ -40,27 +40,35 @@ 'relation': True, } +STAT_TABLE_METRICS = { + 'seq_scan': ('postgresql.seq_scans', AgentCheck.rate), + 'seq_tup_read': ('postgresql.seq_rows_read', AgentCheck.rate), + 'idx_scan': ('postgresql.index_rel_scans', AgentCheck.rate), + 'idx_tup_fetch': ('postgresql.index_rel_rows_fetched', AgentCheck.rate), + 'n_tup_ins': ('postgresql.rows_inserted', AgentCheck.rate), + 'n_tup_upd': ('postgresql.rows_updated', AgentCheck.rate), + 'n_tup_del': ('postgresql.rows_deleted', AgentCheck.rate), + 'n_tup_hot_upd': ('postgresql.rows_hot_updated', AgentCheck.rate), + 'n_live_tup': ('postgresql.live_rows', AgentCheck.gauge), + 'n_dead_tup': ('postgresql.dead_rows', AgentCheck.gauge), + 'vacuum_count': ('postgresql.vacuumed', AgentCheck.monotonic_count), + 'autovacuum_count': ('postgresql.autovacuumed', AgentCheck.monotonic_count), + 'analyze_count': ('postgresql.analyzed', AgentCheck.monotonic_count), + 'autoanalyze_count': ('postgresql.autoanalyzed', AgentCheck.monotonic_count), +} + +IDX_TABLE_METRICS = { + 'idx_scan': ('postgresql.index_scans', AgentCheck.rate), + 'idx_tup_read': ('postgresql.index_rows_read', AgentCheck.rate), + 'idx_tup_fetch': ('postgresql.index_rows_fetched', AgentCheck.rate), +} + # The pg_stat_all_tables contain one row for each table in the current database, # showing statistics about accesses to that specific table. # pg_stat_user_tables contains the same as pg_stat_all_tables, except that only user tables are shown. REL_METRICS = { 'descriptors': [('relname', 'table'), ('schemaname', 'schema')], - 'metrics': { - 'seq_scan': ('postgresql.seq_scans', AgentCheck.rate), - 'seq_tup_read': ('postgresql.seq_rows_read', AgentCheck.rate), - 'idx_scan': ('postgresql.index_rel_scans', AgentCheck.rate), - 'idx_tup_fetch': ('postgresql.index_rel_rows_fetched', AgentCheck.rate), - 'n_tup_ins': ('postgresql.rows_inserted', AgentCheck.rate), - 'n_tup_upd': ('postgresql.rows_updated', AgentCheck.rate), - 'n_tup_del': ('postgresql.rows_deleted', AgentCheck.rate), - 'n_tup_hot_upd': ('postgresql.rows_hot_updated', AgentCheck.rate), - 'n_live_tup': ('postgresql.live_rows', AgentCheck.gauge), - 'n_dead_tup': ('postgresql.dead_rows', AgentCheck.gauge), - 'vacuum_count': ('postgresql.vacuumed', AgentCheck.monotonic_count), - 'autovacuum_count': ('postgresql.autovacuumed', AgentCheck.monotonic_count), - 'analyze_count': ('postgresql.analyzed', AgentCheck.monotonic_count), - 'autoanalyze_count': ('postgresql.autoanalyzed', AgentCheck.monotonic_count), - }, + 'metrics': STAT_TABLE_METRICS, 'query': """ SELECT relname,schemaname,{metrics_columns} FROM pg_stat_user_tables @@ -68,17 +76,12 @@ 'relation': True, } - # The pg_stat_all_indexes view will contain one row for each index in the current database, # showing statistics about accesses to that specific index. # The pg_stat_user_indexes view contain the same information, but filtered to only show user indexes. IDX_METRICS = { 'descriptors': [('relname', 'table'), ('schemaname', 'schema'), ('indexrelname', 'index')], - 'metrics': { - 'idx_scan': ('postgresql.index_scans', AgentCheck.rate), - 'idx_tup_read': ('postgresql.index_rows_read', AgentCheck.rate), - 'idx_tup_fetch': ('postgresql.index_rows_fetched', AgentCheck.rate), - }, + 'metrics': IDX_TABLE_METRICS, 'query': """ SELECT relname, schemaname, @@ -89,6 +92,38 @@ 'relation': True, } +# Toast version of relation and index metrics +# Toast statistics are only available in the stat_all tables as they are in the +# pg_toast schema +TOAST_REL_METRICS = { + 'descriptors': [('relname', 'table'), ('toast_of', 'toast_of'), ('schemaname', 'schema')], + 'metrics': STAT_TABLE_METRICS, + 'query': """ +SELECT s.relname, c.relname as toast_of, schemaname, {metrics_columns} + FROM pg_stat_all_tables AS s + LEFT JOIN pg_class c ON s.relid = c.reltoastrelid + WHERE c.relname = ANY(SELECT relname FROM pg_class WHERE {relations}) +""", + 'relation': True, +} + +TOAST_IDX_METRICS = { + 'descriptors': [ + ('relname', 'table'), + ('toast_of', 'toast_of'), + ('schemaname', 'schema'), + ('indexrelname', 'index'), + ], + 'metrics': IDX_TABLE_METRICS, + 'query': """ +SELECT s.relname, c.relname as toast_of, schemaname, indexrelname, {metrics_columns} + FROM pg_stat_all_indexes AS s + LEFT JOIN pg_class c ON s.relid = c.reltoastrelid + WHERE c.relname = ANY(SELECT relname FROM pg_class WHERE {relations}) +""", + 'relation': True, +} + # The catalog pg_class catalogs tables and most everything else that has columns or is otherwise similar to a table. # For this integration we are restricting the query to ordinary tables. @@ -136,6 +171,7 @@ WHERE {relations}""", 'relation': True, } + # adapted from https://wiki.postgresql.org/wiki/Show_database_bloat and https://github.com/bucardo/check_postgres/ TABLE_BLOAT_QUERY = """ SELECT @@ -243,7 +279,15 @@ 'relation': True, } -RELATION_METRICS = [LOCK_METRICS, REL_METRICS, IDX_METRICS, SIZE_METRICS, STATIO_METRICS] +RELATION_METRICS = [ + LOCK_METRICS, + REL_METRICS, + IDX_METRICS, + TOAST_REL_METRICS, + TOAST_IDX_METRICS, + SIZE_METRICS, + STATIO_METRICS, +] class RelationsManager(object): diff --git a/postgres/tests/common.py b/postgres/tests/common.py index c1c1337274f9e..547e2684b78ec 100644 --- a/postgres/tests/common.py +++ b/postgres/tests/common.py @@ -112,7 +112,7 @@ def check_common_metrics(aggregator, expected_tags, count=1): def check_db_count(aggregator, expected_tags, count=1): aggregator.assert_metric( - 'postgresql.table.count', value=5, count=count, tags=expected_tags + ['db:{}'.format(DB_NAME), 'schema:public'] + 'postgresql.table.count', value=6, count=count, tags=expected_tags + ['db:{}'.format(DB_NAME), 'schema:public'] ) aggregator.assert_metric('postgresql.db.count', value=5, count=1) diff --git a/postgres/tests/compose/resources/03_load_data.sh b/postgres/tests/compose/resources/03_load_data.sh index 424997f191853..1c3adbcb590c9 100755 --- a/postgres/tests/compose/resources/03_load_data.sh +++ b/postgres/tests/compose/resources/03_load_data.sh @@ -12,6 +12,11 @@ psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" datadog_test <<-EOSQL INSERT INTO pgtable (lastname, firstname, address, city) VALUES ('Cavaille', 'Leo', 'Midtown', 'New York'), ('Someveryveryveryveryveryveryveryveryveryverylongname', 'something', 'Avenue des Champs Elysees', 'Beautiful city of lights'); CREATE TABLE pg_newtable (personid SERIAL, lastname VARCHAR(255), firstname VARCHAR(255), address VARCHAR(255), city VARCHAR(255)); INSERT INTO pg_newtable (lastname, firstname, address, city) VALUES ('Cavaille', 'Leo', 'Midtown', 'New York'), ('Someveryveryveryveryveryveryveryveryveryverylongname', 'something', 'Avenue des Champs Elysees', 'Beautiful city of lights'); + CREATE TABLE test_toast (id SERIAL, txt TEXT); + insert into test_toast (txt) select string_agg (md5(random()::text),'') as dummy from generate_series(1,5000); + update test_toast set txt = txt; + SELECT * FROM test_toast; + SELECT * FROM test_toast; SELECT * FROM persons; SELECT * FROM persons; SELECT * FROM persons; diff --git a/postgres/tests/test_relations.py b/postgres/tests/test_relations.py index 4dba8ac940af8..3d03990aef726 100644 --- a/postgres/tests/test_relations.py +++ b/postgres/tests/test_relations.py @@ -5,75 +5,87 @@ import pytest from datadog_checks.base import ConfigurationError -from datadog_checks.postgres.relationsmanager import RelationsManager +from datadog_checks.postgres.relationsmanager import ( + IDX_METRICS, + REL_METRICS, + SIZE_METRICS, + STATIO_METRICS, + RelationsManager, +) from .common import DB_NAME, HOST, PORT -RELATION_METRICS = [ - 'postgresql.seq_scans', - 'postgresql.seq_rows_read', - 'postgresql.rows_inserted', - 'postgresql.rows_updated', - 'postgresql.rows_deleted', - 'postgresql.rows_hot_updated', - 'postgresql.live_rows', - 'postgresql.dead_rows', - 'postgresql.heap_blocks_read', - 'postgresql.heap_blocks_hit', - 'postgresql.toast_blocks_read', - 'postgresql.toast_blocks_hit', - 'postgresql.toast_index_blocks_read', - 'postgresql.toast_index_blocks_hit', - 'postgresql.vacuumed', - 'postgresql.autovacuumed', - 'postgresql.analyzed', - 'postgresql.autoanalyzed', -] - -RELATION_SIZE_METRICS = ['postgresql.table_size', 'postgresql.total_size', 'postgresql.index_size'] - -RELATION_INDEX_METRICS = [ - 'postgresql.index_scans', - 'postgresql.index_rows_fetched', # deprecated - 'postgresql.index_rel_rows_fetched', - 'postgresql.index_blocks_read', - 'postgresql.index_blocks_hit', -] - -IDX_METRICS = ['postgresql.index_scans', 'postgresql.index_rows_read', 'postgresql.index_rows_fetched'] +INDEX_FROM_REL_METRICS = ['postgresql.index_rel_scans', 'postgresql.index_rel_rows_fetched'] +INDEX_FROM_STATIO_METRICS = ['postgresql.index_blocks_read', 'postgresql.index_blocks_hit'] -@pytest.mark.integration -@pytest.mark.usefixtures('dd_environment') -def test_relations_metrics(aggregator, integration_check, pg_instance): - pg_instance['relations'] = ['persons'] +def _get_metric_names(scope): + for metric_name, _ in scope['metrics'].values(): + yield metric_name - posgres_check = integration_check(pg_instance) - posgres_check.check(pg_instance) - expected_tags = pg_instance['tags'] + [ - 'port:{}'.format(pg_instance['port']), - 'db:%s' % pg_instance['dbname'], - 'table:persons', +def _get_oid_of_relation(relation): + oid = 0 + with psycopg2.connect(host=HOST, dbname=DB_NAME, user="postgres", password="datad0g") as conn: + with conn.cursor() as cur: + cur.execute("select oid from pg_class where relname='{}'".format(relation)) + oid = cur.fetchall()[0][0] + return oid + + +def _check_relation_metrics(aggregator, pg_instance, relation): + relation = relation.lower() + base_tags = pg_instance['tags'] + ['port:{}'.format(pg_instance['port']), 'db:%s' % pg_instance['dbname']] + expected_tags = base_tags + [ + 'table:{}'.format(relation), 'schema:public', ] - expected_size_tags = pg_instance['tags'] + [ - 'port:{}'.format(pg_instance['port']), - 'db:%s' % pg_instance['dbname'], - 'table:persons', - 'schema:public', + oid = _get_oid_of_relation(relation) + expected_toast_tags = base_tags + [ + 'toast_of:{}'.format(relation), + 'table:pg_toast_{}'.format(oid), + 'schema:pg_toast', + ] + expected_toast_index_tags = base_tags + [ + 'toast_of:{}'.format(relation), + 'table:pg_toast_{}'.format(oid), + 'index:pg_toast_{}_index'.format(oid), + 'schema:pg_toast', ] - for name in RELATION_METRICS: + for name in _get_metric_names(REL_METRICS): + if name in INDEX_FROM_REL_METRICS: + aggregator.assert_metric(name, count=0, tags=expected_tags) + else: + aggregator.assert_metric(name, count=1, tags=expected_tags) + aggregator.assert_metric(name, count=1, tags=expected_toast_tags) + + for name in _get_metric_names(STATIO_METRICS): + if name in INDEX_FROM_STATIO_METRICS: + aggregator.assert_metric(name, count=0, tags=expected_tags) + else: + aggregator.assert_metric(name, count=1, tags=expected_tags) + # statio table already provides toast specific metrics + aggregator.assert_metric(name, count=0, tags=expected_toast_tags) + + for name in _get_metric_names(IDX_METRICS): + # 'persons' db don't have any indexes + aggregator.assert_metric(name, count=0, tags=expected_tags) + # toast table are always indexed + aggregator.assert_metric(name, count=1, tags=expected_toast_index_tags) + + for name in _get_metric_names(SIZE_METRICS): aggregator.assert_metric(name, count=1, tags=expected_tags) - # 'persons' db don't have any indexes - for name in RELATION_INDEX_METRICS: - aggregator.assert_metric(name, count=0, tags=expected_tags) - for name in RELATION_SIZE_METRICS: - aggregator.assert_metric(name, count=1, tags=expected_size_tags) +@pytest.mark.integration +@pytest.mark.usefixtures('dd_environment') +def test_relations_metrics(aggregator, integration_check, pg_instance): + pg_instance['relations'] = ['persons'] + posgres_check = integration_check(pg_instance) + posgres_check.check(pg_instance) + _check_relation_metrics(aggregator, pg_instance, 'persons') @pytest.mark.integration @@ -119,25 +131,37 @@ def test_relations_metrics_regex(aggregator, integration_check, pg_instance): posgres_check = integration_check(pg_instance) posgres_check.check(pg_instance) - expected_tags = {} for relation in relations: - expected_tags[relation] = pg_instance['tags'] + [ - 'port:{}'.format(pg_instance['port']), - 'db:%s' % pg_instance['dbname'], - 'table:{}'.format(relation.lower()), - 'schema:public', - ] + _check_relation_metrics(aggregator, pg_instance, relation) - for relation in relations: - for name in RELATION_METRICS: - aggregator.assert_metric(name, count=1, tags=expected_tags[relation]) - # 'persons' db don't have any indexes - for name in RELATION_INDEX_METRICS: - aggregator.assert_metric(name, count=0, tags=expected_tags[relation]) +@pytest.mark.integration +@pytest.mark.usefixtures('dd_environment') +def test_toast_relation_metrics(aggregator, integration_check, pg_instance): + pg_instance['relations'] = ['test_toast'] + posgres_check = integration_check(pg_instance) + posgres_check.check(pg_instance) + + oid = _get_oid_of_relation('test_toast') + expected_toast_tags = pg_instance['tags'] + [ + 'port:{}'.format(pg_instance['port']), + 'db:%s' % pg_instance['dbname'], + 'toast_of:test_toast', + 'table:pg_toast_{}'.format(oid), + 'schema:pg_toast', + ] + expected_toast_index_tags = expected_toast_tags + [ + 'index:pg_toast_{}_index'.format(oid), + ] + # 5000 iterations * 32 bytes / 1996 bytes per page = 80.16032064128257 + aggregator.assert_metric('postgresql.seq_scans', count=1, value=1, tags=expected_toast_tags) + aggregator.assert_metric('postgresql.index_rel_scans', count=1, value=3, tags=expected_toast_tags) + aggregator.assert_metric('postgresql.live_rows', count=1, value=81, tags=expected_toast_tags) + aggregator.assert_metric('postgresql.rows_inserted', count=1, value=81, tags=expected_toast_tags) + aggregator.assert_metric('postgresql.index_rel_rows_fetched', count=1, value=81 * 2, tags=expected_toast_tags) - for name in RELATION_SIZE_METRICS: - aggregator.assert_metric(name, count=1, tags=expected_tags[relation]) + aggregator.assert_metric('postgresql.index_rows_read', count=1, value=81 * 2, tags=expected_toast_index_tags) + aggregator.assert_metric('postgresql.index_rows_fetched', count=1, value=81 * 2, tags=expected_toast_index_tags) @pytest.mark.integration @@ -147,14 +171,17 @@ def test_max_relations(aggregator, integration_check, pg_instance): posgres_check = integration_check(pg_instance) posgres_check.check(pg_instance) - for name in RELATION_METRICS: + for name in _get_metric_names(REL_METRICS): relation_metrics = [] for m in aggregator._metrics[name]: if any(['table:' in tag for tag in m.tags]): relation_metrics.append(m) - assert len(relation_metrics) == 1 + if name in INDEX_FROM_REL_METRICS: + assert len(relation_metrics) == 1 + else: + assert len(relation_metrics) == 2 - for name in RELATION_SIZE_METRICS: + for name in _get_metric_names(SIZE_METRICS): relation_metrics = [] for m in aggregator._metrics[name]: if any(['table:' in tag for tag in m.tags]): @@ -179,7 +206,7 @@ def test_index_metrics(aggregator, integration_check, pg_instance): 'schema:public', ] - for name in IDX_METRICS: + for name in _get_metric_names(IDX_METRICS): aggregator.assert_metric(name, count=1, tags=expected_tags)